{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-27T08:22:14.951059Z",
     "start_time": "2020-07-27T08:22:13.249169Z"
    },
    "init_cell": true
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.set_autosave_interval(2000)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Autosaving every 2 seconds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>.container { width:100% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%autosave 2\n",
    "\n",
    "\n",
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:100% !important; }</style>\"))\n",
    "\n",
    "from tqdm.notebook import tqdm\n",
    "import numpy as np\n",
    "from fastai.text import *\n",
    "from modules.callbacks import *\n",
    "from pathlib2 import Path\n",
    "import pickle\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.nn.utils.rnn import pack_padded_sequence\n",
    "from fastai.callbacks import lr_finder, SaveModelCallback, EarlyStoppingCallback,ReduceLROnPlateauCallback\n",
    "import math, copy, time\n",
    "from torch.autograd import Variable\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn\n",
    "seaborn.set_context(context=\"talk\")\n",
    "from modules.model import *\n",
    "\n",
    "defaults.device = torch.device('cuda')\n",
    "#CUDA_VISIBLE_DEVICES=\"\"\n",
    "device =  torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-27T08:22:15.062449Z",
     "start_time": "2020-07-27T08:22:14.952483Z"
    },
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "## Writing processed data into pickle file\n",
    "\n",
    "#pickle.dump(qs, open('fr-2-en-data/fr-en-qs.pkl','wb'))\n",
    "qs = pickle.load(open('fr-2-en-data/fr-en-qs.pkl','rb'))\n",
    "\n",
    "en_qs,fr_qs = zip(*qs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-07-27T08:22:13.249Z"
    },
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "path = Path()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-07-27T08:22:13.249Z"
    },
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "en_tok = pickle.load((path/'fr-2-en-data/en_tok.pkl').open('rb'))\n",
    "fr_tok = pickle.load((path/'fr-2-en-data/fr_tok.pkl').open('rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-07-27T08:22:13.250Z"
    },
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "TMP_PATH = path/'fr-2-en-data/tmp/'\n",
    "\n",
    "def load_ids(pre):\n",
    "    ids = np.load(TMP_PATH/f'{pre}_ids.npy',allow_pickle=True)\n",
    "    itos = pickle.load(open(TMP_PATH/f'{pre}_itos.pkl', 'rb'))\n",
    "    stoi = collections.defaultdict(lambda: 3, {v:k for k,v in enumerate(itos)})\n",
    "    return ids,itos,stoi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-07-27T08:22:13.251Z"
    },
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "en_ids,en_itos,en_stoi = load_ids('en')\n",
    "fr_ids,fr_itos,fr_stoi = load_ids('fr')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fastai's Data Bunch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-07-27T08:22:13.252Z"
    },
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "def seq2seq_collate(samples, pad_idx=1, pad_first=True, backwards=False):\n",
    "    \n",
    "    # unpack samples to tuples\n",
    "    samples = to_data(samples)\n",
    "    \n",
    "    # find max len of x, y batch wihich decides inp seq length\n",
    "    max_len_x,max_len_y = max([len(s[0]) for s in samples]),max([len(s[1]) for s in samples])\n",
    "    \n",
    "    max_len = max(max_len_x,max_len_y)\n",
    "    \n",
    "    # create a dummy tensor of height batch_size and width max_len with padded value\n",
    "    res_x = torch.zeros(len(samples), max_len).long() + pad_idx\n",
    "    res_y = torch.zeros(len(samples), max_len).long() + pad_idx\n",
    "    \n",
    "    # if backwards activate reverse mode used in bi-directional   \n",
    "    if backwards: pad_first = not pad_first\n",
    "    \n",
    "    # fill vocabulary indices\n",
    "    for i,s in enumerate(samples):\n",
    "        if pad_first: \n",
    "            res_x[i,-len(s[0]):],res_y[i,-len(s[1]):] = LongTensor(s[0]),LongTensor(s[1])\n",
    "        else:         \n",
    "            res_x[i, :len(s[0])],res_y[i, :len(s[1])] = LongTensor(s[0]),LongTensor(s[1])\n",
    "    \n",
    "    # flip backward if backwards = True\n",
    "    if backwards: res_x,res_y = res_x.flip(1),res_y.flip(1)\n",
    "        \n",
    "    res_x_mask = (res_x != pad_idx).unsqueeze(-2)\n",
    "    res_y_mask = None\n",
    "    \n",
    "    # target mask creation\n",
    "    if res_y is not None:\n",
    "        dec_y = res_y[:, :-1]\n",
    "        tar_y = res_y[:, 1:]\n",
    "        decode_lengths = torch.tensor([len(s[1]) for s in samples])-1\n",
    "    return (res_x[:,1:],dec_y,decode_lengths), tar_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-07-27T08:22:13.252Z"
    },
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "class Seq2SeqDataBunch(TextDataBunch):\n",
    "    \" decorator adds following method additionally to base class 'TextDataBunch\"\n",
    "    \n",
    "    @classmethod\n",
    "    def create(cls, train_ds, valid_ds, test_ds=None, path:PathOrStr='.', bs:int=8, val_bs:int=None, pad_idx=1,\n",
    "               dl_tfms=None, pad_first=False, device:torch.device=device, no_check=True, backwards:bool=False, **dl_kwargs)-> DataBunch:\n",
    "        \n",
    "        \"Function takes pytorch dataset object transforms into 'databunch' for classification and cls will allow to access parent class methods just  like 'self'\"\n",
    "        \n",
    "        # store dataset obj into list\n",
    "        datasets = cls._init_ds(train_ds,valid_ds,test_ds)\n",
    "        val_bs = ifnone(val_bs, bs) #returns val_bs if none bs\n",
    "        \n",
    "        # stores raw seq of unequal len into tensor with padding\n",
    "        # below function takes batches output from Dataloader returns padded tensor\n",
    "        collate_fn = partial(seq2seq_collate, pad_idx=pad_idx, pad_first=pad_first, backwards=backwards)\n",
    "        \n",
    "        #sampler function: generater takes dataset then sorts and returns sorted indices    \n",
    "        train_sampler = SortishSampler(datasets[0].x, key=lambda t: len(datasets[0][t][0].data), bs=bs//2)\n",
    "        \n",
    "        # train data loader obj with Sortish sampler(sort with some randomness)\n",
    "        train_dl = DataLoader(datasets[0], batch_size=bs, sampler=train_sampler, drop_last=True, **dl_kwargs)\n",
    "        dataloaders = [train_dl]\n",
    "        \n",
    "        # other dataloaders with pure sorting append into dataloaders list\n",
    "        for ds in datasets[1:]:\n",
    "            lengths = [len(t) for t in ds.x.items]\n",
    "            sampler = SortSampler(ds.x, key = lengths.__getitem__)\n",
    "            dataloaders.append(DataLoader(ds,batch_size=val_bs,\n",
    "                                          sampler = sampler,**dl_kwargs))\n",
    "        \n",
    "        return cls(*dataloaders, path = path, device = device, collate_fn=collate_fn, no_check = no_check)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-07-27T08:22:13.253Z"
    },
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "class Seq2SeqTextList(TextList):\n",
    "    _bunch = Seq2SeqDataBunch\n",
    "    _label_cls = TextList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-07-27T08:22:13.254Z"
    },
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:100% !important; }</style>\"))\n",
    "\n",
    "pd.set_option('display.width', 10000)\n",
    "pd.set_option('display.expand_frame_repr', True)\n",
    "\n",
    "\n",
    "data = pd.DataFrame(qs, columns=['en', 'fr'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-21T05:45:35.652983Z",
     "start_time": "2020-05-21T05:45:35.625698Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_4aefb0da_9b26_11ea_b144_393a8fa313efrow0_col0 {\n",
       "            width:  300px;\n",
       "        }    #T_4aefb0da_9b26_11ea_b144_393a8fa313efrow0_col1 {\n",
       "            width:  300px;\n",
       "        }    #T_4aefb0da_9b26_11ea_b144_393a8fa313efrow1_col0 {\n",
       "            width:  300px;\n",
       "        }    #T_4aefb0da_9b26_11ea_b144_393a8fa313efrow1_col1 {\n",
       "            width:  300px;\n",
       "        }    #T_4aefb0da_9b26_11ea_b144_393a8fa313efrow2_col0 {\n",
       "            width:  300px;\n",
       "        }    #T_4aefb0da_9b26_11ea_b144_393a8fa313efrow2_col1 {\n",
       "            width:  300px;\n",
       "        }    #T_4aefb0da_9b26_11ea_b144_393a8fa313efrow3_col0 {\n",
       "            width:  300px;\n",
       "        }    #T_4aefb0da_9b26_11ea_b144_393a8fa313efrow3_col1 {\n",
       "            width:  300px;\n",
       "        }    #T_4aefb0da_9b26_11ea_b144_393a8fa313efrow4_col0 {\n",
       "            width:  300px;\n",
       "        }    #T_4aefb0da_9b26_11ea_b144_393a8fa313efrow4_col1 {\n",
       "            width:  300px;\n",
       "        }</style><table id=\"T_4aefb0da_9b26_11ea_b144_393a8fa313ef\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >fr</th>        <th class=\"col_heading level0 col1\" >en</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_4aefb0da_9b26_11ea_b144_393a8fa313eflevel0_row0\" class=\"row_heading level0 row0\" >7957</th>\n",
       "                        <td id=\"T_4aefb0da_9b26_11ea_b144_393a8fa313efrow0_col0\" class=\"data row0 col0\" >En quoi consiste une inspection d'un agent des pêches?</td>\n",
       "                        <td id=\"T_4aefb0da_9b26_11ea_b144_393a8fa313efrow0_col1\" class=\"data row0 col1\" >What happens if I'm inspected by a fishery officer?</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_4aefb0da_9b26_11ea_b144_393a8fa313eflevel0_row1\" class=\"row_heading level0 row1\" >23052</th>\n",
       "                        <td id=\"T_4aefb0da_9b26_11ea_b144_393a8fa313efrow1_col0\" class=\"data row1 col0\" >Quels sont les avantages et les inconvénients du système de La Haye pour un utilisateur tel que vous?</td>\n",
       "                        <td id=\"T_4aefb0da_9b26_11ea_b144_393a8fa313efrow1_col1\" class=\"data row1 col1\" >What are the pluses/minuses of the Hague system for you as an individual user?</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_4aefb0da_9b26_11ea_b144_393a8fa313eflevel0_row2\" class=\"row_heading level0 row2\" >34628</th>\n",
       "                        <td id=\"T_4aefb0da_9b26_11ea_b144_393a8fa313efrow2_col0\" class=\"data row2 col0\" >À qui ces activités douteuses profitent-elles?</td>\n",
       "                        <td id=\"T_4aefb0da_9b26_11ea_b144_393a8fa313efrow2_col1\" class=\"data row2 col1\" >Who profits from this dubious activity?</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_4aefb0da_9b26_11ea_b144_393a8fa313eflevel0_row3\" class=\"row_heading level0 row3\" >29141</th>\n",
       "                        <td id=\"T_4aefb0da_9b26_11ea_b144_393a8fa313efrow3_col0\" class=\"data row3 col0\" >Quelles sont les forces et faiblesses du programme de la PNE et des activités qui y sont associées?</td>\n",
       "                        <td id=\"T_4aefb0da_9b26_11ea_b144_393a8fa313efrow3_col1\" class=\"data row3 col1\" >What are the strengths and weaknesses of the NCB Program and its activities?</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_4aefb0da_9b26_11ea_b144_393a8fa313eflevel0_row4\" class=\"row_heading level0 row4\" >22353</th>\n",
       "                        <td id=\"T_4aefb0da_9b26_11ea_b144_393a8fa313efrow4_col0\" class=\"data row4 col0\" >• Que fait l'OMPI en plus de traiter les affaires soumises en application des principes UDRP?</td>\n",
       "                        <td id=\"T_4aefb0da_9b26_11ea_b144_393a8fa313efrow4_col1\" class=\"data row4 col1\" >What is WIPO doing in addition to administering cases under the UDRP?</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7efb12012f50>"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.sample(5)[['fr', 'en']].style.set_properties(subset=['fr', 'en'], **{'width': '300px'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-22T04:13:26.212097Z",
     "start_time": "2020-05-22T04:13:26.006457Z"
    }
   },
   "outputs": [],
   "source": [
    "!jupyter nbextension enable --py widgetsnbextension --sys-prefix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-07-27T08:22:13.254Z"
    },
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "# lowercase\n",
    "data['en'] = data['en'].apply(lambda x:x.lower())\n",
    "data['fr'] = data['fr'].apply(lambda x:x.lower())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### create a Seq2SeqTextList (Fastai's datablock api)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-07-27T08:22:13.255Z"
    },
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "# text list with 'fr' column as label  and split list randomly (0.8) \n",
    "src = (Seq2SeqTextList.from_df(data, path = path, cols='fr')\n",
    "       .split_by_rand_pct(seed=42)\n",
    "       .label_from_df(cols='en',label_cls=TextList))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-07-27T08:22:13.256Z"
    },
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "src = src.filter_by_func(lambda x,y: len(x) > 30 or len(y) > 30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-07-27T08:22:13.257Z"
    },
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "from modules.model import *\n",
    "from modules.callbacks import *\n",
    "from transformer.model import Translation\n",
    "\n",
    "en_emb = torch.load('models/en_emb.pth')\n",
    "fr_emb = torch.load('models/fr_emb.pth')\n",
    "\n",
    "databunch = src.databunch()\n",
    "databunch.batch_size = 64\n",
    "\n",
    "'''\n",
    "(Pdb) dec_input\n",
    "['[SOS]', 'Bible', 'Color', '##ing', \"'\", 'is', 'a', 'coloring', 'application', 'that', 'allows', 'you', 'to', \n",
    "'experience', 'beautiful', 'stories', 'in', 'the', 'Bible', '.', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', \n",
    "'[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', \n",
    "'[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
    "\n",
    "(Pdb) dec_output\n",
    "['Bible', 'Color', '##ing', \"'\", 'is', 'a', 'coloring', 'application', 'that', 'allows', 'you', 'to', \n",
    "'experience', 'beautiful', 'stories', 'in', 'the', 'Bible', '.', '[EOS]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', \n",
    "'[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', \n",
    "'[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]']\n",
    "\n",
    "-> dec_inputs.append(dec_input)\n",
    "\n",
    "(Pdb) dec_input\n",
    "[1, 5907, 13068, 1160, 114, 1112, 172, 25889, 4050, 1117, 3645, 1130, 1108, 2543, 2714, 2803, 1109, 1105, \n",
    "5907, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
    "\n",
    "(Pdb) dec_output\n",
    "[5907, 13068, 1160, 114, 1112, 172, 25889, 4050, 1117, 3645, 1130, 1108, 2543, 2714, 2803, 1109, 1105, \n",
    "5907, 121, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
    "'''\n",
    "\n",
    "\n",
    "\n",
    "'''{'kor_vocab_length': 50000,\n",
    " 'eng_vocab_length': 28998,\n",
    " 'd_model': 768,\n",
    " 'd_ff': 2048,\n",
    " 'd_k': 64,\n",
    " 'd_v': 64,\n",
    " 'num_layers': 12,\n",
    " 'num_heads': 8,\n",
    " 'start_word': '[SOS]', #xxbos\n",
    " 'end_word': '[EOS]',\n",
    " 'sep_word': '[SEP]',\n",
    " 'cls_word': '[CLS]',\n",
    " 'pad_word': '[PAD]', #xxpad\n",
    " 'mask_word': '[MASK]'} #xxpad 0'''\n",
    "\n",
    "'''model = Translation(src_vocab_size=len(fr_stoi),\n",
    "    tgt_vocab_size=len(en_stoi),\n",
    "    d_model=300, d_ff=2048,\n",
    "    d_k=64, d_v=64, n_heads=6, \n",
    "    n_layers=6, src_pad_index=src_pad_index,\n",
    "    tgt_pad_index=src_pad_index, device=device)'''\n",
    "\n",
    "#model = make_model(len(fr_stoi), len(en_stoi), N=6, h = 6, d_model=300,src_embedding = None,tar_embedding = None).to(device);\n",
    "model = Translation(src_vocab_size=len(fr_stoi),\n",
    "    tgt_vocab_size=len(en_stoi),\n",
    "    d_model=300, d_ff=2048,\n",
    "    d_k=64, d_v=64, n_heads=6, \n",
    "    n_layers=6, src_pad_index=1,\n",
    "    tgt_pad_index=1, device=device)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-13T04:16:06.968618Z",
     "start_time": "2020-06-13T04:16:06.952009Z"
    }
   },
   "outputs": [],
   "source": [
    "dict(src_vocab_size=len(fr_stoi),\n",
    "    tgt_vocab_size=len(en_stoi),\n",
    "    d_model=300, d_ff=2048,\n",
    "    d_k=64, d_v=64, n_heads=6, \n",
    "    n_layers=6, src_pad_index=1,\n",
    "    tgt_pad_index=1, device=device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Fastai - with CrossEntropyLoss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-07-27T08:22:13.257Z"
    },
    "hidden": true,
    "init_cell": true
   },
   "outputs": [],
   "source": [
    "def CrossEntropy_loss(input,targets):\n",
    "    pred,_,_,_,decode_lengths = input\n",
    "    x,y = pred.contiguous().view(-1, pred.size(-1)), targets.contiguous().view(-1)\n",
    "    loss = nn.CrossEntropyLoss().to(device)(F.log_softmax(x, dim=-1), y)\n",
    "    return  loss #loss(pred.data.long(), targets.data.long())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-01T16:56:28.698814Z",
     "start_time": "2020-07-01T16:56:28.616475Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from modules.callbacks import *\n",
    "opt_fn = partial(optim.Adam, betas=(0.9, 0.98))\n",
    "\n",
    "#criterion = LabelSmoothing(len(en_itos), 1, 0.1).to(device)\n",
    "\n",
    "#learn = Learner(dataBunch,arch,loss_func= loss_func,opt_func=opt_fn, metrics=[topK_accuracy,BleuMetric(metadata,vocab)],callback_fns=[ShowGraph]) #,TeacherForcingTurnOff,TeacherForcingCallback\n",
    "\n",
    "\n",
    "\n",
    "learn_CEL = Learner(databunch,model,loss_func=CrossEntropy_loss,wd=0.1, metrics=[seq2seq_acc, CorpusBLEU(len(en_itos))],callback_fns=[ShowGraph]) #,TeacherForcingTurnOff,TeacherForcingCallbacklearn = "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-01T16:33:02.751747Z",
     "start_time": "2020-07-01T16:32:45.880112Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='1', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/1 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>seq2seq_acc</th>\n",
       "      <th>bleu_metric</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='89' class='' max='435', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      20.46% [89/435 00:16<01:02 20.2933]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n",
      "Min numerical gradient: 3.02E-05\n",
      "Min loss divided by 10: 1.20E-02\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEZCAYAAACJjGL9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd5hU9dnG8e8z2ztllw7SQZqCooBYY+waE40FRbFiexM1mpi8xmiq0RijsVesQVFe0dgVOwIi0kQ6SC+7sLtsb7/3jzOL67oMW2b2zO7en+uaa90zZ848M4N7zzm/Zs45RERE9ibgdwEiIhLdFBQiIhKSgkJEREJSUIiISEgKChERCSnW7wLCycwq8MIv3+9aRERakHSgyjlXZyZYa+oea2ZVgGVkZPhdiohIi5GXlwfgnHN1XmVqVWcUQH5GRkZGbm6u33WIiLQY7dq1Iy8vb69XYtRGISIiISkoREQkJAWFiIiEpKAQEZGQFBQiIhKSgkJEREJqbd1jG+2D5dvZnl9C/05p9O+USkZSnN8liYhEBQVF0H/mrOedpdv2/N4pLYG+WSm0T44nJSGWlPgYUhNj6dYuiT6ZKfTJTKFLeiJVDnIKS9meX8qO3aUkx8fQJyuFrNQEzMzHVyQiEh4KiqCMpDg6pSWwfXcpANt3l+75771JiA1QUeWorPrh6PaUYGCM7t2BK4/qR6e0xIjULSISaa1tCo/cpo7MzisuZ/WOAlZtK2BtTiG7S8opKq2koLSC/JJyNuwsZnNeMXW9bfExAcoqq36wPTk+hssO78tlR/QlNUHZLCLRJTgyO885166u+xUUjVBSXsn6nUV8m1NEYlyAzumJdEpLICMpjpLyKtblFLI2u5BlW/J5Zva37CoqByAzNZ7Lj+jL2L6ZDO6aRlyM+hKIiP8UFD7LLynn4Y9W8/inaykp/+5sIyE2wLDuGRzQox2DuqQysHMaAzqn6YxDRJqdgiJKbM0r4d6ZK5n5zXa25pfsdb/eHZO55PC+nDO6p844RKRZKCii0Ja8Yhasz2XBhly+3pzPim27f9Bw3iczhRuPH8SJw7qo95SIRJSCooXILSpj+dbdvDx/Iy99uZHqjlQjemQwqld7stISyEpNoFN6AiN7ticjWeM8RCQ8FBQt0PKtu7nz7WW89832Ou8PGIzs1Z4jB2Zx1KAshnfP0FmHiDSagqIFm7t2JzMWbGJbfik7CkrJ3l3KtvwSKmqN2xjbtyP3nHMgndI1VkNEGk5B0coUl1Uye00OH63YwYfLt7MupwiAzNQE7j33QMb1y/S5QhFpaRQUrdw7X2/lhmkLyS+pIGBw3bEDufro/gQCuhQlIvWzr6BQ/8sW7rihXXj9F4czvHsGVQ7uencF5z02h3XZhX6XJiKthIKiFejZIZmXrhzLBWP3A+DzNTkc/6+PefDD1ZTXMaWIiEhD6NJTKzNz2TZu/r8lbM7zBvXt3zWdv58xnBE96jyjFBHRpae25pjBnXnn+iOZNK43ZvDNlnx+9sAsHv5oNVV1zHIrIrIvvgaFmR1mZjPNrNDM8s3sDTMb7mdNrUFqQiy3njaU6VeOY0CnVCqqHH97cxkXTfmC7ILQU6eLiNTmW1CY2RjgA7w1Mc4DJgGZwMdm1t+vulqTkb3a8+o14zn3kJ4AfLRiByfd8wmzVmX7XJmItCS+tVGY2TvAMKCfc644uK0dsAZ40zl3XiOO2ebbKPbm1YWb+d30xRSUet1obz55CBcd1lsjukUkqtsoxgIzq0MCwDmXC3wCnG5mMb5V1gqddkA3Xv/FeIZ1T6fKwR//u5TfTl9MWYV6RYlIaH4GRTxQ1wXzUiAZ6Fv7DjPLDXUDMiJcc4u2X8cUpk0ex8nDuwIw9YsNTHx8DjsLy3yuTESimZ9BsRQYYzWufZhZHHBI8FfNRREBSfEx3DdhJNceOwCAOWt38pP7P2Xltt0+VyYi0crPoPg3MAS418y6m1lP4BGgR/D+H1wTcc61C3UD8pqv/JbLzLj22IHcP2EUiXEBNuws5mcPzOLjFTv8Lk1EopBvQeGcewK4Ca+300ZgPTAI+Edwl83+VNZ2nDyiK9Mmj6NTWgK7Syu4aMoXPDP7W7/LEpEo4+s4Cufc3/EuMQ0HejvnxgEdgG+dcxv8rK2tGN4jgxnXHMbQbulUVjl+/8oS/jBjCYWlFX6XJiJRIqqm8DCz3sBC4E/OuX+E3rvOx6t7bCMVllZw7QsLeHfpNgAykuK4cOx+XDiuNx1TE3yuTkQiKWqnGTezA4DTgXl4PZ0OBH4LzAdOdM41+CutgqJpqqoc/3p/JQ9/tJrSYLfZxLgAZx/ck+uPG0RGkpZfFWmNojkoBuM1Xg8DUvAG2j0N/NM516h5JhQU4ZFTUMpTs9bx1OffkldcDsDAzqlMuegQurVL8rk6EQm3qA2KSFBQhFdhaQXPzv6Wf7yznPJKR+f0BKZcdAj7d033uzQRCaNoHpktUS4lIZbJR/bjqYsOIS0hlm35pfz8oc/5dKXmihJpSxQUsk/j+mcy7cqxdM1IpKC0gklPzuWDZdv9LktEmomCQuplcJd0pl81jsFd0qioctz8yhKKyyr9LktEmoGCQuqta0YSD088iPiYAJtyi3nk4zV+lyQizUBBIQ2yX8cULjm8DwAPfrSKzbnF+3iEiLR0CgppsKuP7k9WWgIl5VX8/a1lfpcjIhGmoJAGS02I5dfHDwJgxoLNzFu30+eKRCSSFBTSKGeM6sGIHt7yH7e9tpSqqtYzHkdEvk9BIY0SCBh/OHUoAIs35THtS83hKNJaKSik0Q7arz2nH9gNgD//9xs27CzyuSIRiQQFhTTJzacMITPVW8/i+hcXUKlLUCKtjoJCmiQzNYF//HwEAF+s28WDH67yuSIRCTcFhTTZUYM6MWlcbwDufm8lCzZoUkaR1kRBIWFx04mDGdQ5jcoqxy+nfqUV8kRaEQWFhEViXAz3nHsg8bEBvs0p4i9vfON3SSISJgoKCZvBXdL3DMSbOnc9a7MLfa5IRMJBQSFhdeG43vTskESVg3vfX+l3OSISBgoKCau4mAC/OGYAADMWbGLV9t0+VyQiTaWgkLD76cju9O6YTI+dW9h83sWQng6BgPfzqqtg9Wq/SxSRBtCa2RIRs+57hgOvu4zYqgriq2oscBQX591eeglOPNG/AkVkj32tma2gkPBbvRo3YgRWFGJKj+RkWLQI+vVrvrpEpE77CgpdepLwu+surLw89D7l5XD33c1Tj4g0ic4oJPzS02F3PRqx09MhLy/y9YhISDqjkOZXUBDe/UTEVwoKCb/U1PDuJyK+UlBI+J1/vtezKZS4OJg4sXnqEZEmUVBI+P3qV/ULiuuua556RKRJFBQSfv36eeMkkpN/EBhlgRjKE5K8+9U1VqRFUFBIZJx4ojdO4vLL94zMLk1O5T8HnMBJl9zH1nFH+12hiNSTusdKsykoreCoOz8ku6CUk4Z34e6zDyQhNsbvskTaPHWPlaiRmhDLDccNBOCNxVs57u6PeWvJFlrTlxWR1khBIc3qrIN7MvmIvsQGjG9zirji2fmc88hslmzSwDuRaKVLT+KL1TsK+Ovr3/D+su0AxAaMm0/enwvH9cbMfK5OpG3RpIAS1T5dmc2tr33Nqu3eKO0zRvXgLz8dRmKc2i5EmovaKCSqjR+QyavXHMbJI7oC8PL8jZz18Odszi32uTIRqaagEN8lx8dy37kjuenEwQQMFm3M47T7PmPDzhDTlItIs1FQSFQwM644sh9TLjqE9MRYsgtK+dW0hVRWtZ5LoyItlYJCosoRA7O455yRAMxdu5PHP13jc0UioqCQqHP04E6cd2gvAP7x9gq+2ZLvc0UibZuCQqLS/568P707JlNWWcV1LyygtKJy3w8SkYhQUEhUSo6P5Z9nH0jAYNnW3fzznRV+lyTSZikoJGqN6tWea47uD8Ajn6xh7tqdPlck0jYpKCSq/c+PBjC8ewbOwW+nL9IlKBEfKCgkqsXFBLj9jOHEBIzVOwp56EP1ghJpbr4GhZmNNLNXzGyzmRWa2VIzu8nMEvysS6LL0G4ZXDq+DwD3f7Bqz3QfItI8fAsKMxsMzAJ6A9cCpwLTgb8Aj/pVl0SnXx47gJ4dkiirrOJ3/7eYKg3EE2k2fp5RnAMkAmc45150zs10zt0MPA+ca2b7WHRZ2pLk+Fj+fPpwwBuIN+3LDT5XJNJ2+BkU5cGftRciyAvep1ZL+Z4jB2bxkwO7AfDXN5axPb/E54pE2gY/g+IZYCfwoJn1MbN0M/sJcCFwl3OuysfaJEr9/pQhZCTFkVdczoTH5rA1T2EhEmm+BYVzbj0wBhgCrME7k3gFuNc59/u6HmNmuaFuQEazvQDxRWZqAneffQDxMQFWbS/gzIdmsS670O+yRFo1Pxuz9wNeA3YAPwWOBm4FfmVmf/KrLol+xwzuzJMXjSY5PoaNu4o586HPNR+USAT5tsKdmf0HOALo75wrrrH9D8AtQD/n3LoGHlMr3LUh89fv4qInvyCvuJz0xFieuvgQRvZq73dZIi1ONK9wNxJYWjMkgubh1TW4+UuSlmRUr/a8MHkMWWkJ5JdUcMHjc1m0UV8SRMLNz6DYDAwzs+Ra28cGf25q5nqkBRrcJZ1pk8fSOT2B3aUVnP/YHJZsqt2RTkSaws+guBfoArxtZmeY2bFm9mfg18B7zrnFPtYmLUjvzBSev+y7M4vzH5+jNguRMGpwUJhZfzM7oda2Q83sNTP7zMwur89xnHOvAD8GSoEHgBl4jdp/Ak5vaF3StvXLSuX5Sw+lY0o8uUXlnPfYHFZs2+13WSKtQoMbs83sZaCDc+7o4O+ZwAogFSgO/jwjGATNSo3Zsnzrbs555HN2FZWTmZrAC5PH0C8r1e+yRKJaJBqzDwbeq/H7uUA6MArIAuYAv2zEcUWabFCXNJ67dAwZSXFkF5Qy4dHZfJujcRYiTdGYoMjCa4iudgLwmXNuiXOuDJiKN4hOxBdDuqXzzCWHkJYQy7b8UiY8OoeNu4r8LkukxWpMUBQC7QDMLAYYD3xc4/5ivDMMEd+M6NGOKRcfQnJ8DJtyi5nwqKb7EGmsxgTF18BEM+sIXIbXJvFujfv3wxttLeKrg/ZrzxOTRpMYF2D9ziIunvIFZRWaQkykoRoTFHcCI4DtwP3AV8AnNe4/Dpjf9NJEmm5M3448MvFgzGDplnzufX+l3yWJtDgNDgrn3OvAMcC/gNuA41yw61TwLGMjMCWMNYo0yREDs5h8RD8AHvhwFfPX7/K5IpGWxbe5niJB3WNlb0orKvnJfZ+xbOtu+mSm8MYvDicpPsbvskSiQrPM9WRmscHR1ZeZWZdwHFMknBJiY7jrrAOIizHWZhdy+5vf+F2SSIvRmJHZd5jZFzV+N7xxFS8CDwOLzaxf+EoUCY+h3TK49tiBADz1+bd8ujLb54pEWobGnFGcwPcbr0/Fmy78TmBCcNtNTaxLJCImH9GXUb28s+tfv7SQorIKnysSiX6NCYqeQM2uI6cCa51zNznnpgIPAT8KR3Ei4RYbE+Cusw4kPjbA5rwS7n1/ld8liUS9xgRFPFBZ4/ej+f6UHmuArk0pSiSS+mSmcNVR3tXRxz5Zw0pNHigSUmOCYgPeWteY2VCgL/BRjfs7AQVNL00kcq44sh/7dUymosrx+xlLaE29/0TCrTFBMRW40Mz+C/wXyAfeqHH/SGB1GGoTiZjEuBhuO20oALPX7GTGgs37eIRI29WYoPgb3oC6sYADLnDO5QKYWQZwGvB+uAoUiZSjBnXixGFeb+4/v/4NecXlPlckEp0aMzK71Dl3iXOuo3Our3Pu1Rp378Zrn7g1XAWKRNLvTxlCcnwM2QWl/POd5X6XIxKVwroUqnOuyjmX55zTVzNpEbq1S+LaYwcA8PRsja0QqUujgsLMUszsNjNbZGYFwdsiM7vVzFLCXaRIJF10WB8O6NkO5+DaF75ie76mIxepqTEjszsAc4HfA13wZo/9CugM3ALMDe4j0iLExQS479yRpCfGkl1Qxi+mfkVllXpBiVRrzBnFH4HBwDVAV+fc4c65w4FuwNXAINRGIS1Mzw7J/OPnBwBeL6h7NB25yB6NCYrTgMeccw845/YMvHPOVTrnHgSeAE4PV4EizeW4oV24ZHwfAP49c6XaK0SCGhMUnfEuNe3N/OA+Ii3Ob04YvKe94pdTv2JddqHfJYn4rjFBsQ1vUN3ejAzuI9LixMd67RUZSXHkFJZx3mNz2JRb7HdZIr5qTFC8BlxiZpPNbM/jzSxgZpcDFwOv7vXRIlGuZ4dknrxoNCnxMWzKLea8R2erJ5S0aQ1e4S643OnnQD9gB1A9SmkQkAWsAsY553LCWGd9a9MKdxI2s9fkcOETcymtqGJg51SmXj6WDinxfpclEnZhX+EuGAAHA7cDOcDo4C0bb3qPg/0ICZFwG9O3Iw9PPIi4GGPFtgIueGIOhaVav0LankYNuHPO5Tvn/tc5N9Q5lxy8DXPO3QxMMLOlYa5TxBdHDerEv88dRUzAWLIpnz++pn/a0vaEdQqPoEy8y1AircIJw7rw2xMHA/DCvA28tWSrzxWJNK9IBIVIq3PxYX0Y3z8TgJumL2KbGrelDVFQiNRDIGDcddYBtEuOI7eonBumLaRK03xIG6GgEKmnzumJ3P6z4QB8sjKbJz5b63NFIs1DQSHSACcM68rZB/cE4I63lrNwg7piS+sXW5+dzOz6BhzzsEbWItIi3HLqEOaszWFdThEXT/mCl64cR59Mza4vrVe9BtyZWVUDj+ucczGNK6nxNOBOmsuq7QWc+dAscovK6dkhiZevHEentES/yxJplH0NuKtvUBzZ0Cd2zn3U0Mc0lYJCmtNX63cx4dE5FJdXsn/XdF6YPIb0xDi/yxJpsLAERUuhoJDm9sHy7Vz61Dwqqxxj+nbgqYsPISG22U+mRZok7FN4iMh3jh7UiTvOGAF4Cx7dOG2Rus1Ksyspr2TavA1s3x2Z8T0KCpEmOuOgHvzmBG/k9qsLN3P3eyt8rkjamtlrcrjxpUWM/dtM8kvKw358BYVIGFxxZF/OPaQXAP+euYpp8zb4XJG0JR8u3wHAAT0yItJOpqAQCQMz448/GcrhA7xpPn47fTGzVmkpVYk85xwzl20HvEuhkaCgEAmTuJgAD5w3isFd0qiockx+9ktWbd/td1nSyq3NLmT9ziIAjh6soBCJemmJcTw+aTRZaQnsLqng6ue+oqS80u+ypBX7IHjZKSstgSFd0yPyHAoKkTDr3i6JhyceREzAWL5tN7e/uczvkqQV+3C5d9npyIFZBAIWkedQUIhEwKhe7bnu2AEATJm1jpnLtvlckbRGRWUVzFmzE4hc+wQoKEQi5sqj+nNInw4A3DhtUcT6uEvbNWtVDmWVVcQEjPHBjhSR4GtQmNkUM3Mhbl38rE+kKWICxt1nH0h6Yiw5hWXcoMF4EmYfBC87HbRfezKSIjd9jN9nFH8Cxta6HQEUA3Occ1pzUlq07u2S+GtwDYuPV+zg3pkraU3T5oh/nHN7xk9E8rIT+BwUzrnVzrnZNW9AFpAEPO5nbSLhcsqIbvz8oB4A/Ou9ldz8yhIqKhs6IbPI963cXsCm3GIAjh6cFdHn8vuMoi4XA0XAC34XIhIufzp9GKeM6ArAc3PWc+nT8ygorfC5KmnJPggOsuuakcigzmkRfa6oCgoz6wqcALzknMv3ux6RcEmMi+Hec0Zy9dH9AG/KhTMfnMWWvGKfK5OWqrp94qhBWZhFpltstagKCuBCIIa9XHYys9xQNyCjWasVaYBAwLjx+MH8/YzhxAaMZVt3c+ETcymt0IA8aZjdJeXMW7cLgKMi3D4B0RcUk4BVzrmP/S5EJFLOHt2LJyaNJiZgrNhWwP0frPa7JGlhPluVTUWVIy7GOKx/5LrFVouaoDCz8cAg4Mm97eOcaxfqBuQ1W8EiTXDEwCwuO7wvAA98sIpvtuhKq9Tfl996ZxMje7YnNSE24s8XNUGB14hdCTzldyEizeHaYwfQJzOFiirHb15epJ5QUm9Lg18shnVvnqvtUREUZpYC/Bx42zm3ye96RJpDYlwMtwfHWCzamMeTn63ztyBpEZxzLN3sBcWQbpGZBLC2qAgK4GwgFXjC70JEmtOhfTty/hhvwaO73l3OuuxCnyuSaLctv5RdRd4qdpGaLba2aAmKi4Bs4FW/CxFpbr85YTDdMhIpKa/i1y/pEpSEtnSL1xQbF2P075TaLM8ZFUHhnDvcOZflnAv/Yq8iUS4tMY6/BC9BzV23kz+//o3PFUk0q77sNKBTGvGxzfMnPCqCQqStO3pQJ646yhuMN2XWOl74Yr3PFUm0qm7Ibq72CVBQiESNG44bxLH7e4Onbn5lCfPW7fS5IolGexqym6l9AhQUIlEjEJyWfECnVMorHVc8++WeSd9EAApKK1iX462PrTMKkTYqLTGORy84mIykOLILyrj86XkUlWnyQPEsqzEwc3+dUYi0Xb0zU7h/wihiAsbXm/O5/oWFWvBIgO/aJ3q0T4roQkW1KShEotD4AZnccsoQAN76eit3vbvc54okGvjRPgEKCpGodeG43lwwdj8A7v9gNdPnb/S5IvFb9RlFc152AgWFSFS75ZQhHD7Amx30ppcXqydUG1ZRWcWyrbuB5m3IBgWFSFSLjQlw34RR9MtKoayyisuensenK7P9Lkt8sCa7kLIKb9S+Lj2JyPdkJMXxxKTRdEiJZ1dROROfmMM/31lOpRq425Tq9om0xFh6tE9q1udWUIi0APt1TGHG1YdxQI8MnIN7Z65iwqOz2ZZf4ndp0kz2jMjumh7xpU9rU1CItBA9OyQz7YpxXHxYHwDmrN3JSfd8wqKNuT5XJs3hGx+m7qimoBBpQeJjA9xy6hAenngQ6Ymx5BSWMeHROXyhRu5W7XtrUDRz+wQoKERapOOHdmH6VYfRJT2RgtIKJj4+h09W7vC7LImQ7btLySksA3RGISIN0L9TKtOuGEvPDkmUlFdxyZR5vP31Vr/LkgioPpuIizEGdEpr9udXUIi0YD07JDNt8rg93Wevem4+byze4ndZEmbVDdn9m3ENipoUFCItXJeMRF6cPJYhXdOprHL84j9f8d7SbX6XJWG0cIPXYcGP9glQUIi0Ch1TE3ju0kMZ3CWNiirHVc/N56MVarNoDZxzLAgGxYG92vlSg4JCpJVonxLPs5ceuucy1OVPz+Pz1Tl+lyVNtCWvhO27SwEY2VNBISJNlJmawPOXjaF3x2RKK6q45Kkv9nwblZap+vNLiA0wqEvzN2SDgkKk1emcnsjzl42hR/skisoqufq5+ewKdq2Ulqc6KIZ3zyAuxp8/2QoKkVaoW7sknpw0mqS4GDblFnP9iwu0+FELtWB9sH3Cp8tOoKAQabUGdE7jrz8bBsAHy3fw8MdrfK5IGqqisorFm/IA/xqyQUEh0qr9dGQPzhndE4B/vLOcuWs11UdLsnzbborLKwGdUYhIBN162lAGd0mjssrxP/+ZT3ZBqd8lST1Vt09kpibQvV3zTi1ek4JCpJVLjIvhgfNGkRIfw7b8Uq56bj6lFZV+lyX1ULN9ormnFq9JQSHSBvTNSuXOnx8AwNy1O/nd9CU4p8btaLcwOIX8SB/bJ0BBIdJmnDS8KzccNxCAl+dv5IEPV/tckYSyu6ScldsLAH/bJ0BBIdKmXH10f342qjsAd769XBMIRrHFG/NwDsxgRI8MX2tRUIi0IWbG3342nEN6dwDguhcWaOR2lPoq+Ln0z0olLTHO11oUFCJtTEJsDA9NPIj9gtN8XPrUF2zYWeR3WVLLnokAfb7sBAoKkTapQ0o8T0waTUZSHNkFZUx6ci65RZrmI1pEw4yxNSkoRNqoflmpPHrBwcTHBFi9o5DLn/lS3WajxOa8EnYEZ4zVGYWI+OqQPh2466zvus3eOG2R5oSKAtXjJ5LiYhjU2Z8ZY2tSUIi0cace0I2bThwMwKsLN3PXu8t9rqhtc84xc9l2wJsxNtanGWNr8r8CEfHd5CP6ct6hvQC4/4PVvKlus77IKy7nime/5OX5GwE4rH+mzxV5FBQigplx22lDGdu3IwA3TFvIqu27fa6qbVm4IZeT7/2Et7/21jv/6cjuTD6yr89VeRQUIgJAbEyAf08YSdeMRArLKpn8zJfsLin3u6w2Yerc9Zz50Cw27iomITbAHWeM4J9nHUBiXIzfpQEKChGpITM1gQfOG7WnJ9SN0xZpTqgIe/Kztdw0fTHllY5+WSnMuOYwzhrd09dJAGtTUIjI94zs1Z5bTxsKwFtfb+XBjzQnVKQ89skabnttKQCH9e/Iq9eMZ3CXdJ+r+iEFhYj8wLmH9OSsg3sAcMdby5mxYJPPFbU+D3+0mj+//g0Ahw/I5PELR5OSEOtzVXVTUIjID5gZf/zJMEb3bg94jdsfr9jhc1WtxyMfr+Zvby4D4KhBWTx6wcFR0x5RFwWFiNQpMS6Gxy4YzaDOaZRXOq549ksWagLBJluyKY/bgyHxo8GdeHjiQVEdEqCgEJEQMpLjePqSQ+jeLomiskoumvIFq3cU1Pvx7y7dxg3TFvLM5+tYuW13m28Yr6is4rfTF1PlYECnVB44fxQJsdEdEgDWmj44M8vNyMjIyM3Vtx6RcFqzo4AzH/qcnYVldG+XxItXjN3nGs4fLNvOpU/Po7LGlCAdU+I5tG8HDuufyfj+mfTqkBxVvXsi7bFP1uxpl3jpirEcHJzu3W/t2rUjLy8vzzlX58RSvgeFmR0F/A44BIgH1gH/cs490ohjKShEImThhlzOfXQ2RWWV9MlM4YXJY+iUlljnvl+t38WER+dQXF5J93ZJlFVW7ZnkrqYe7ZMY27cj7ZLjCASMGDNiYwLs1yGZod3T6Z+VGhVTWITDxl1F/PifH1NcXsl5h/biLz8d7ndJe0R1UJjZhcDjwKPADKAcGBys675GHE9BIRJBn6/OYdKTcymtqGJg51SmXj6WDinx39tn9Y4CznxwFruKyunZIYmXrxxHVmoCa7ILmb0mh1mrcpi1OptdRfsezBcfG2BwlzTG9O3I8UM7M7Jne13OBPIAAA9oSURBVAKBlncG4pzjkqfmMXPZdrLSEnjv+iPJSPJ3MaKaojYozKwnsBy41Tl3R5iOqaAQibAPl2/nsqfnUV7pGNotnecvG0NGUhzOOTbuKuacR2azKbeYjinxvHTlOPpkpvzgGFVVjqVb8vl0VTYL1udSWlFJpfO2l1ZUsmJbAXnFPwySTmkJ/HhIZ44Z3InRfTqQ7vPKb/X130Wbueb5rwB44LxRnDS8q88VfV80B8WtwK+Bjs654jAdU0Eh0gze/norVz03n8oqR2ZqPGDsKirb0x6RHB/D1MvHMKJH49ZScM6xOa+ErzflsWhjHu99s41lW78/91TAvNlVx/TryHFDOnPQftFxvb+2DTuLOP3+z8gpLOPY/Tvx6AUHR127TDQHxUwgA/gncAvQH9gCPAvc4pz7wXJbZravBMjIyMhAQSESeTMWbOLaFxZQ+09ISnwMD5x/EEcOzArr863LLuSdpVt5++ttLNiQ+71GcvDGI/z6+MEM6RY9I5t3FZZxxoOzWJNdSEZSHG/88vB9dgLwQzQHxTKgG1AB/B74GjgG+C3wonPuvDoeo6AQiSKLN+Yx79uddEiJp31yPB1S4unVMTnil4QKSiuYt24nn6/J4aPlO/acbZjBTw7oxjXH9KdvZqqv7Rkl5ZWc99gcvvx2F/GxAZ679FBGR0kvp9qiOShWAAOAc51zU2tsvxO4ARjgnFvVwGPq0pNIG+Oc492l27jz7eWs3P7dGI+E2AB9MlPok5nCgE6pHNirHSN7tqd9rcb32vKKy/lk5Q7W7yxi465iNu0qJqewlHH9Mpl8RF86pibss6bKKsdVz33J219vwwzunxB97RI1RXNQfA6MATo453bV2H4M8D5wtnPuxQYeU0Eh0kZVVjlenr+Rf727gs15JXvdr29mCiN7tefAXu04sEc7BndNI8aM2WtyeHHeBt5cspXSiqo6H5sSH8PF4/tw6eF96+y1tKuwjEWb8pg+fyMzFmwG4JZThnDx+D7heZEREs1B8QhwGT8Mih8B7wE/d8691MBjKihE2riKyiq+3VnEmh2FrM0uYM2OQpZuyWfp5nwq6lgPPCE2QEZSHNtrjPNIjAswoFMa3dsl0aN9ErExAf4zd/2enljpibGM7NWe6jZp52BtdiHrdxZ979iXju/DzacMidyLDZNoDooTgDeB851zz9XYfhdwHdDHOfdtA4+poBCROhWXVbJ4Ux7z1+/iq/W7WLghj6353z/zOGi/9px9cE9OGtGV1FozueYVl/P4J2t4/NO1FJZV7vV5AgYDO6dx4rCu/M8x/VvEuI+oDQoAM3sDGAf8ge8as38NPOKcu6oRx1NQiEi9bc0rYcGGXDbnFnPEwEz6d0rb52NyCkqZ9uVGdhV+1zHTAZ3TEzmgRwZDu2WQFB/98zfVFO1BkQLcBpwLZAHrgceAO5xzdV8kDH08BYWISAPtKyh8XSXDOVeI18PpBj/rEBGRvWsds22JiEjEKChERCQkBYWIiISkoBARkZAUFCIiEpLvK9yFk5lVAZaRkeF3KSIiLUZeXh6Ac87VefLQ2oKiAu8sKd/vWlqQ6lTN87UKqYs+m+jV2j6bdKDKOVfnkIlWFRTScNVTt+9toI34R59N9Gprn43aKEREJCQFhYiIhKSgEBGRkBQUIiISkoJCRERCUlCIiEhICgoREQlJ4yhERCQknVGIiEhICgoREQlJQSEiIiEpKFoQM+thZveY2admVmBmzsyOitBzTTCzhWZWYmYbzex2M0ustc+kYA113RL3duzWKAo/m9PM7AMz22pmpWa2xcxeM7MxkagpmkXhZzPUzB40s7nB/ZyZ9Y5EPeGioGhZ+gPnAgXA+5F6EjM7H3gO+Aw4EfgrcDUwZS8PuQAYW+tWGqn6olS0fTaZwBfANcBxwHVAFvCJmY2PVH1RKto+m4OBU4GtwX2jn3NOtxZyAwI1/vt0wAFHhfk5YoAtwIxa2y8LPt+hNbZNCm470O/3xu9btH02e3l8Ol6AP+H3+9WWP5ta9VwbvL+33+9TqJvOKFoQ51xVffYzs0Qz+6OZraxx2eE+M0urx8PHAF2Ap2ptfw4oB85oWNVtQwv5bArwgqK8PrW2FtH22dS3nmhS5yIV0nKZWQzwOnAQ8DdgHjAY+BMw3MyO3sc/1GHBn0tqbnTOFZnZ6hr31/SWmWUBucC7wM3OuVVNeyWtjx+fTfA5A0B34LeAAQ808aW0Oj79f9NiKChan7OBY4CTnXNvBLe9b2YbgVfwrp2+HuLxHYM/d9Zx384a94N3jfUvwGxgNzAK74/RHDMb7Zxb0+hX0To152dTbQ7eHz/wLo2c4Jxb2NDC2wA/PpsWQ5eeWp+TgBzgHTOLrb7hfdOvBI4E7xtUzfvNrPa/hb0N2d+z3Tn3lnPuZufcf51zHznn7sb7ny0V+F24X1gr0GyfTQ0TgUPxLn0sAd6MVI+fFs6Pz6bFUFC0Pp3xvr2U17oV4jW4ZQb3e7/W/U8Et+cEf9b1DagDdX9j2sM5twT4Eq/nk3xfs382zrlvnHNznXPT8b4VrwHuCceLaWV8/f8m2unSU+uTDWwDTglxP8BkIK2O7V8Hfw4DVlTfaWbJQD/gtXrUEABaXINdM/D1s3HOVZrZl8CEhpXdJkTD/zdRS0HR+ryJd721yjk3f287OeeW7+Wu2XhtDxOB6TW2nwvE1dr2A2Y2DBgJPN2AmtsKvz+bBGAcoI4GP+TrZxPtFBQtjJmdGfzP0cGfR5pZJlDonHsTeB5vANxbZnY33mUgB/QEjgfucc7N2tvxnXMVZnYTMMXM7gNeAvYH/g685JybXaOWd4GZeN+mCvAC4jdAHl4jd5sShZ/NR3jtErnAfsAVwEDgZ2F6yS1GlH02yXhtIgAHBH+eaGY7gB3OuY+a/orDzO+BHLo17Ib3j7eu27oa+8QDNwGLgRK8P9yLgX8BXev5POcHH1MKbALuAJJq7XM3Xkjk412vXQ88BvTy+33SZ8NtwHxgV/Cz2Qr8HzDe7/dJnw29Q9Tzod/vVV03rUchIiIhqdeTiIiEpKAQEZGQFBQiIhKSgkJEREJSUIiISEgKChERCUlBIdIEZtY7uJTlrX7XIhIpCgppdmZ2VPCP6w1+19JamNmt9v01y6vMbKeZvW9mp4Xp+KeHo1ZpeTSFh0jTfAskARV+FxJ0C7AW7//tfniT2M0ws/Odc8814bh/wFu97ZWmlygtjYJCJMjM0pxzuxvyGOdNbVASoZIa403n3LzqX8zsJWAB3tQUTQkKacN06UmimpklmNnvzOxrMysxs1wze83MRtbaL2Bm/2tmH5vZVjMrM7P1ZvagmXWste+edgUzO9vMvjSzYuDfwfunBO/PCD5+e/C5PzOzQ/d2rL0c/xQz+yL4+C1mdmdwQZzar/MMM1sY3G+9mf3BzI4NHmdSY98/561mlw0MqOM5rzKzd8xsU/D92mJmz5pZ79qvJfjrhTUvb9U61rHBY+UGX8MiM7uisXVLdNEZhUQtM4sD3sKbGvsZ4D4gA7gM+MzMjqjx7TkeuBF4GZiBt+DMaOASYLyZHeScK6v1FKcDvwAeBB7Cm9ywpreBHcAf8RakuR54w8x61/PM4yTgquCxnwB+AtyAN1HfX2u8zrOB/wCr8SbzqwAuBE6tx3OEZGbtgfbA9jruvgFveux78RbWGQZcChxjZsOdczl4r38i3vv/CfBIHc9xefA1zsabNbgQ+DHwoJn1c87d2NTXIT7ze1ZC3dreDTgKb6bMG/ax33XB/Y6vtT0db6baD2tsM2rN0hncfknwGGfV2NY7uK0c2L+Ox0wJ3v9Are0/D26fXMexbq1jWyHQu1aNS4AtNbbF4s0yug1oX2N7Kt5qdA6YVI/39Nbgvj/CW42tC3AY8EFw+x11PCaljm0/Cu7/61rbHTCljv274l16e76O++7BW0a0n9//5nRr2k2XniSanQ8sA740s8zqG97Zw7t4ZwpJ4LUVOOeKYc+6xu2C+84MHuvQOo7/unPumxDPf3et36uP9YPLOHvxinNuXfUvzvvr+QHQxcxSg5sPArrh/RHeVWPfArxv6Q31Ht5ZwBbgU7wlaf9OHWuYO+cKYc9lu4zg+7UQb3rtut6vupwJJACP1/yMgsd6De/y9o8a8TokiujSk0Sz/fF6FO0IsU8msAHAzM4CfoW3gFJcrf3a1/HYFXVsq2lNzV+cczlmBnWvi7zPxwfVXFu5AOgT/L2uldP2tppaKFfjva5k4Gi8S2vtnXM/6JVlZsfg9ZI6FEisdXdd71dd9g/+fC/EPp3reSyJUgoKiWaGtwjM9SH22QFgZj8DXgDmAr/EC48SIAavnaOus+eiUE/unKsMUVd97O3xNY9R32PV11z3XbvNq2a2DfibmX3lnNtzhmJmo4F38JZFvQmvS20x3iWmqdS/o0t1/RfgncXUpa7AlBZEQSHRbCWQBcx0zlXtY9+JeMFwtHNuTwCY2eAI1hcOa4M/B9VxX13bGuouvHaaP5vZ88656gb7CXgheqJzrroGzCyF+p9NgPcZAWQ750KdVUgLpjYKiWZP4zXK1nlGYWY1L2lU4n0bDtS434CbI1lgGMzD+yY+KdhDCYBgG0aTu5c658rxelh1xLsMVa36bKf2Gc3vqPvvQgHQoY7tL+It+3lbdXtRTcG2j4SG1i3RRWcU4qcfmVnta+PgfTt9CK/XzI+BO4PX02fidWHthddAWoJ3HR68xezPAGaa2dN4bRSn412rj1rOuYrgVCbPAXPN7HG87rGT8Noz+uAFYFM8g9cWcb2Z/ds5l4e3fvZ1eN19HwHK8N7rEXjjLmqbDRxrZr/B63HmnHNTnXMbzexKvLXSvzGzZ/BGq2cBw/E+gyHAuia+BvGRgkL8dELwVtty4CHnXLmZnYw3FmEi3hgDgM14bRFPVT/AOTfVzNLw/vj9A2+swmt4199ziGLOuefNrALv7Oc2vK6yjwOLgOl4bQdNOX6Fmd2O14vqWuA259xnZnYG8HvgT8HneA84Evi4jsNcBdwP/C+QFtw2NXj8J81sBd64jMlAO7ywWR48/tam1C/+M6/HnohEGzP7FV7ojXXOzfa7Hmm7FBQiPjOzeKCyZi+rYBvFIrzBhd3cD0eVizQbXXoS8V9f4E0zm4rXC6or3hQefYArFRLiNwWFiP924DUWnwd0wmvMXgzc5Jx70c/CRECXnkREZB80jkJEREJSUIiISEgKChERCUlBISIiISkoREQkJAWFiIiE9P+eK5kLg7XvFAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_CEL.lr_find()\n",
    "learn_CEL.recorder.plot(suggestion=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-01T17:11:48.973219Z",
     "start_time": "2020-07-01T16:56:32.666300Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>seq2seq_acc</th>\n",
       "      <th>bleu</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2.456067</td>\n",
       "      <td>2.653436</td>\n",
       "      <td>0.612640</td>\n",
       "      <td>0.433706</td>\n",
       "      <td>01:28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2.041358</td>\n",
       "      <td>2.261911</td>\n",
       "      <td>0.642574</td>\n",
       "      <td>0.462979</td>\n",
       "      <td>01:28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.675195</td>\n",
       "      <td>1.926699</td>\n",
       "      <td>0.680742</td>\n",
       "      <td>0.488075</td>\n",
       "      <td>01:28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1.384582</td>\n",
       "      <td>1.713956</td>\n",
       "      <td>0.702641</td>\n",
       "      <td>0.511958</td>\n",
       "      <td>01:31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.127888</td>\n",
       "      <td>1.588813</td>\n",
       "      <td>0.723198</td>\n",
       "      <td>0.536749</td>\n",
       "      <td>01:33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.813250</td>\n",
       "      <td>1.529009</td>\n",
       "      <td>0.734520</td>\n",
       "      <td>0.553977</td>\n",
       "      <td>01:37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.497641</td>\n",
       "      <td>1.541128</td>\n",
       "      <td>0.743082</td>\n",
       "      <td>0.570197</td>\n",
       "      <td>01:32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.262595</td>\n",
       "      <td>1.580004</td>\n",
       "      <td>0.747232</td>\n",
       "      <td>0.581183</td>\n",
       "      <td>01:31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.140268</td>\n",
       "      <td>1.620333</td>\n",
       "      <td>0.750187</td>\n",
       "      <td>0.587652</td>\n",
       "      <td>01:31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.086930</td>\n",
       "      <td>1.639049</td>\n",
       "      <td>0.750771</td>\n",
       "      <td>0.589219</td>\n",
       "      <td>01:32</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEDCAYAAAAlRP8qAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxU1cH/8c+ZJSvZCFsg7FZcUMQKtYobSFuXutDFurRqRX8+3dxaa/u4UJdK7fK01m5aKdhV2qrVWrVacUHBrSKLgCKbQCAkkMmeTGbO74872SdkJpnkzgzf9+s1zMy9d+49czJ8c3Lm3HONtRYREUl9HrcLICIiiaFAFxFJEwp0EZE0oUAXEUkTCnQRkTThc+OgxpgWnF8m1W4cX0QkReUDYWtt1Ow2bgxbNMaEAZObl4/PYwb9+CIiqSgQCABYa23U3hVXWuhAtcnMLXh4+QbOOrrEpSKIiKSWwsJCAoFAjz0brvahW3RSk4hIorgb6MpzEZGEcbmFLiIiieJyC12RLiKSKBqHLiKSJtwa5QJAWC10kaRgraWiooLGxkbC4bDbxTmoeDwefD4f+fn55Obm9mtfrga68lzEfdZadu7cSU1NDZmZmXi9XreLdFAJBoPU19dTVVVFXl4eo0ePxuPpW+eJAl3kIFdRUUFNTQ0jR45k6NChbhfnoBQOh6msrKSiooJAIEBRUVGf9qNRLiIHucbGRjIzMxXmLvJ4PAwbNoyMjAxqa2v7vp8EliluGuUi4r5wOKxuliRgjMHn8/XrOwy10EVE0oS7wxaV6CIiCeNqoGvYoohI4qjLRUTS1sqVK1mwYAFVVVUJ37cxhgULFiR8v/2hyblEJG2tXLmS733vewMS6CtWrGD+/PkJ329/uDsOXW10EUkC4XCYUCiE3++P+TXHH3/8AJaob9RCF5G0tGDBAq677joAJk6ciDEGYwxbt27FGMO1117Lz372Mw455BAyMjJ45ZVX2l43Y8YMioqKKCgoYObMmSxdurTb/rt2uSxYsABjDOvXr+eCCy4gPz+fkSNH8uUvf7n1SkMDzuUWuogkq5ZQmLJAo9vFaFNSkIXPG3sbdP78+QQCAX7605/yyCOPUFLiXB2t9f6vf/0rY8aMYeHChQwZMoTJkycDsH37dr761a8yduxYWlpaWLZsGRdddBE1NTVcccUVvR533rx5XHDBBVx55ZWsXr2a7373uwAsWrQo3rccN1cDXU10keRVFmjkpHuWuV2MNi/feBpjh+bEvH1paSnjx48HYPr06UyYMKHT+sbGRp577jny8/M7Le8YvOFwmDlz5lBRUcF9990XU6BfddVVbX8ZnH766XzwwQcsWrSIBx98EGMG9hrKLs+26ObRReRgNmfOnG5hDrBs2TIWLlzIqlWr2Lt3b9sZ7VlZWTHt95xzzun0/Oijj6axsZE9e/YwatSo/hf8AFyenEuJLpKsSgqyePnG09wuRpuSgtgCNeb9lXS/QP3KlSuZO3cus2fP5he/+AVjxozB7/fzq1/9KuYuk+Li4k7PMzMzAecvgoGmPnQRicrn9cTVxZFqonV/PPzww/j9fp544om2IAZobm4ezKL1mUa5iEjaag3lhoaGmLZvnSCr43zk5eXl/OMf/xiQ8iWazhQVkbQ1depUAO677z5WrFjBm2++ecDW9llnnUVtbS0XX3wxzz33HH/4wx846aSTGDly5GAVuV80fa6IpK2TTjqJm266iUceeYRZs2YxY8YMdu3a1eP2c+bM4f777+ftt9/m7LPP5o477uDaa6/l4osvHsRS951xI1SNMVUmM7fg/mdXM/+kSYN+fBFpt23bNoC2IX7int5+FoWFhQQCgYC1tjDaes22KCKSJvSlqIhImtCXoiIiaUItdBGRNOFyC12JLiKSKGqhi4ikCXcvEi0iIgnj7rBFTbcoIpIwMQW6MWa6MeYxY8wuY0ydMeZdY8xNxpjM3l/dM8W5iEji9DrbojHmMOBVYCNwLVABzAbuAo4AvtTXg6sPXUQkcWJpoX8ByAI+Y61daq193lp7M/An4EJjTOxXVe1Co1xEJFVcdtllna561Hpt0sWLF8f92oESy3zowch916ucBiLrQn09uFroIpKqSkpKWLFiRdu1SJNBLIH+e+B64FfGmBuBSuA04FLgx9bacNcXGGOqetlnAagPXURSV2ZmJscff7zbxeik1y4Xa+124Hic/vLNOC3zx4B7rbW39OvoaqKLyAB59NFHMcbw4osvdlt355134vP52LVrF88++yznnHMOY8aMITs7m0MPPZTrr7+e6urqA+6/py6XRYsWMWXKFDIzM5kyZUrMl65LhFi+FB0PPAHsBs4HqoBTgO8YY8LRQr2nqR077LMKKNCoRZEkFmqB6p1ul6Jd/hjwxn7VzLPPPpthw4axePFiTjnllE7rHnroIebOncvo0aN5/PHHmTVrFldddRV5eXm8//773H333bzxxhu8/PLLcRXxwQcfZP78+Zx//vn85Cc/Yf/+/dx2220Eg8FOV0EaKLHUzkIgD5hurW29jtMLkevx3WqMedBau7UvB9eXoiJJrHon/Oxot0vR7prVUBT7nO1+v5+LLrqIRYsWcd9995GbmwvAK6+8wvvvv8+dd94JwNVXX932GmstJ554IlOmTOHkk0/mnXfeYdq0aTEdLxwOc8sttzBz5kz+/ve/t12z9IQTTmDKlCmMGTMm5rL3VSy/MqYD73YI81ZvRl5/WF8Prh4XERlIl19+ObW1tfztb39rW7ZkyRKKioo499xzAdi9ezff+MY3mDhxIllZWfj9fk4++WQANmzYEPOxNm7cSFlZGRdddFGnC1BPmjSJE044IUHv6MBiaaHvAqYaY3KstfUdln88ct/nv8mU5yJJLH+M0ypOFvnxt3CPOeYYpk2bxuLFi7n00ktpaGhg6dKlXHzxxWRmZhIOh5k7dy7l5eXceuutTJ06ldzcXD788EPmzZsX88WlASorKwEYNWpUt3UlJSVtVyMaSLEE+r3Ao8Azxpif4nwpeipwI/CctXZNXw+uFrpIEvP64uriSFaXXnopN9xwA1u3bmXFihUEAgEuu+wyANasWcPatWtZsmQJX/pS+zmSgUDXUdq9Ky4uBpwWf1dlZWV9K3ycYhnl8hgwF2gCfgn8A+fL0TuA8/pzcPWhi8hAu+SSS/D5fCxZsoQlS5Zw5JFHMmPGDIC2rpGMjIxOr3nggQfiPs6UKVMoKSnhT3/6U6flmzdv5tVXX+1j6eMT01fG1trngOcSfnTluYgMsOHDh3PGGWfwm9/8hvLychYuXNi27vDDD2fSpEncdNNNABQUFPDnP/+Zt956K+7jeDwe7rjjDubPn8+8efO44oorqKqq4tZbb6WkpCRh7+eAZRiUo/RAF4kWkcFw+eWXU1ZWhjGGSy65pG253+/n8ccfZ/LkyVx55ZV88YtfxOv18pe//KVPx7niiiv47W9/y9q1a5k3bx633347N998M7Nnz07UWzkgY10IVWNMlcnMLbj9b69z89lHDPrxRaRd65d148enfn95quvtZ1FYWEggEAj0dK6PLhItIpImXA30kE4VFRFJGFcDvSXcbV4vERHpI7XQRUTShLst9JACXUQkUVzuclGgi7jN4/EQCvX5OjWSQOFwuNM8MPFSoIsc5Hw+H8FgkLC+03JVKBSisbGRrKysPu/D5T50fYBE3Jafn084HKayshI3zksRJ8xb53vJy8vr835iny1+AKgPXcR9ubm55OXlUVFRQXV1NT6fq7Fw0AmHwzQ1NWGtZeTIkWRnZ/d5X+4GurpcRJLC6NGjCQQC1NbWqutlkHm9XoYOHUpeXl6/whwU6CKC88VoUVERRUVFbhdF+kF96CIiacLVQA+qD11EJGF0pqiISJrQOHQRkTShPnQRkTShuVxERNKEulxERNKEvhQVEUkTLg9bVB+6iEiiqIUuIpIm1IcuIpImXB7loi4XEZFEcTXQG4Ihzb8sIpIgrgZ6YzBMU4ta6SIiieBqoAM0BnUtQxGRREiCQFcLXUQkEVwP9KYWtdBFRBLB9UBXC11EJDFiDnRjzKnGmH8bY6qMMfXGmHeNMVf1twDqQxcRSYyYrilqjLkUeBB4APgJEAQOAzL6WwAFuohIYvQa6MaYscCvgO9aa+/psOo//Tmwidw3atiiiEhCxNLlckXk/ucDUQC10EVEEiOWQD8ZWA/MM8ZsNMaEjDE7jDELjTFRu1wi/ew93oACE2mi68QiEZHEiKUPfXTk9nPgFmAdMBv4DjAWuLg/BVALXUQkMWIJdA+QB1xorf1LZNkLxphs4JvGmNustZs6vsBaW3igHRpjqgwUADQp0EVEEiKWLpfKyP0zXZY/Fbk/tk9HjvS5aBy6iEhixBLoa3pY3jpQpU+J3DbKRS10EZGEiCXQH4ncn9ll+ZmABd7oTwEadeq/iEhC9NqHbq192hjzFPALY8ww2r8UvQb4tbV2W18O3NpCb1KXi4hIQsR0pijwOeB7wI3AcGA7cDNwz4FedECRRFcLXUQkMWIKdGttHfDNyC0h2vvQ1UIXEUkEF2dbbB3loha6iEgiuBforV0uaqGLiCSEa4He9qWo+tBFRBLC9QtcaJSLiEhiuN5C1ygXEZHESII+dAW6iEgiuNhCdxK9QYEuIpIQ7gV6pIVe3dDiVhFERNKK633ogYYg4bB1qxgiImnD9VEuAJv21rpdBBGRlOdaoHs8pu1xRW2TW8UQEUkbrne5AITU5SIi0m+udrnkZTlzg9U16YtREZH+cjXQczOcQH/xvQo3iyEikhZcDfTd1Y0A5GZ43SyGiEhacDXQzztmNAD76prdLIaISFpwNdBH5GcBsFejXERE+s3VQB9d4AR6WaDRzWKIiKQFVwN9WF4moHHoIiKJ4GqgF+c6gV5VHyQY0rzoIiL94XIfembb493qdhER6RdXA33c0Jy2WRc/3F/vZlFERFKeq4Hu93oYNsRppW/eW+dmUUREUp7rsy2WREa6vL5ln8slERFJba4H+tihOYBGuoiI9JfrgX7i5GEArN4R0KyLIiL94HqgH11aAEBtUwvlNRrpIiLSV64H+oRhuW2P7/znehdLIiKS2lwP9CGZvrbHT64pc7EkIiKpzfVABzj76BIAPEYXuxAR6aukCPTbz50KQNhqPLqISF/1KdCNMQuMMdYYsyoRhSjK8VOY4wfgvT01idiliMhBJ+5AN8YcCXwb2JOoQhhjOGT4EAC279MUACIifRFXoBtjPMCDwG+BDYksyKjIGaOapEtEpG/ibaFfB5QC/5vogowpygZgc0VtonctInJQ8PW+icMYMwm4HbjYWlttWqdJjL5tVS+7K+i6YPrYIgDe+TBAYzBEll8XjhYRiUdMLXTjpPcDwDPW2scGoiAzJjiB3hwK88ZWTdQlIhKvWLtcrgSOA74ey8bW2sID3YBA19cUD8nk0JHOF6O3/WNdjMUSEZFWvQa6MWYYcA9wN1BnjCk0xhTidNd4I8+zElGY86aPAWBzRR31zTrBSEQkHrG00Etx+rzvBvZ3uJ0ITI08XpCIwlxw3Ni2x79YtikRuxQROWjE8qXoJuC0KMt/CgwB5gPbE1GY4iHt1xhdtmEv3/rkYYnYrYjIQaHXQLfW1gIvdF3eOpLFWtttXX9865NT+OEzG3m3rJpAfZCCyBmkIiJyYEkxl0tHsw8b0fb48sWvu1gSEZHUEvM49K6stacmsBxtDhuV1/a4sq55IA4hIpKWkq6FbozhmjkfASBbJxeJiMQs6QIdYNpY50TSzXvraAmFXS6NiEhqSMpAP2S40+3SHAqzTbMviojEJCkDvbQomyy/U7Q5P37R5dKIiKSGpAx0j8e0TdYFsGO/WukiIr1JykAH+O2lx7U9/udqXTxaRKQ3SRvouZk+vvTx8QA8vmqXy6UREUl+SRvoAOdMGw3Au2XVPLRiq6tlERFJdkkd6MeOa+9Hf2jFNhdLIiKS/JI60D0ew1dOnQzApvJaXtlU4XKJRESSV1IHOsA1p3+k7fEXH3wNa62LpRERSV5JH+iZPi83zD0UgLCF65e+43KJRESSU9IHOsDXZh/S9vjRt3dS16SrGYmIdJUSgW6M4brTD217fte/1rtYGhGR5GTc6JM2xlQVFBQUVFVVxfW6CTc92fb4qDEFrNkZ4JLjx3HneUcluogiIkmnsLCQQCAQsNYWRlufEi30Vkv/38fbHq/ZGQDgDyu3c/3Dq9wqkohI0kipQJ85cWjU5Y9E+tWrG4M0BkODXCoRkeSQUl0uAIH6INNu/zcAnzpyFE+v291tm9ULPkF+lq5FKiLppbculz5fgs4tBTl+ti48q+15x371Vn96bTtXnzJ5MIslIuK6lOpyiWbRZcd1W7bwqQ1c9dCbLpRGRMQ9KR/osw8byd3znFEu3/7UYW3L//3uHibc9CSrd/TQrVOzB9Y9CnWaTkBE0kPK9aH35ht/fpvH3+k83e7d847iwpnjOm/434fg8a87j0dOhYknO7fxJ0BWQcLLJSLSX731oaddoAOs3Rng7J8v77Z8y91nYoxxnrz0I3jph9DS2KVwHhg93Qn3CSfBuOMhI3dAyikiEo+DMtABGoMhZv1gGRW1TZ2Wv3zjaYwdmuM8CTbCjjdgy0vObeebEO4yrYDHD6Uz2lvwpceBL3PAyi0i0pODNtBb7apq4ISFz3dbXpybwa2fPoL65hBfmDHWabk31cL2lbDlRSfgy94ButSPL9tptU88GSaeAiXTwJtyg4VEJAUd9IEOsGZHgE/f170LpqOXvnUa44pzOi9s2A/bXm1vwZe/2/2Fmfkw/sT2FvyII8CT8t81i0gSUqBH1Da1cPXv32J5LxfJWPmdOVxw/wq2VdZzeEk+R47OZ/u+eu67cDojPNWw9eX2gN+3ufsOcophwqz2FnzxIdDaby8i0g8K9A6CoTCLlm/hxEOG8dCKrSx9c0dcr99wx6fI8nvbF1R92Dngq3d2f1FeSXvrfeLJUDiu+zYiIjFQoMdgwePrWPzq1l63mz6ukEe/cmL0ldY6LfbWcN/yEtRH+WugaEJ7633CSZA3sl9lF5GDhwI9RuGw5fT/e5HNe+uYOCyXZd88lbe37+fXL37AM+v2tG33k89PY96xpb3v0FooX98e7luXQ1Og+3bDD3O6aEpnwtgZUDRRXTQiEpUCPQEamkMcfuvTbc+Xf/s0SotyDvAKaAmFWbMzQDBknVkiwyFn1ExrwG9fAcH67i/MGeYMkyw9zrkfcyxk5iX6LYlICup3oBtj5gBfBD4OjAX2Aa8Dt1lr1/SlUKkW6AB/fG0b//vo2k7LfnrBMRw/qZgXNpZz0yMHropnrzuZj4zsEMwtzbDzrfZw3/kWNFV3f6HxOCNnSme034oP0UgakYNQIgL9r0AxsBRYD4wEbgSmAqdaa1fGW6hUDHSA0370Alsq6vq1j7duPp2huRntZ6y2CoehYqNzotOON+DDN2DvBrqNgwfIKoy04Gc692M+CtlRf74ikkYSEegjrLXlXZYVAluA5621n4m3UKka6BB9ut7++PUlx9LUEuaUQ4dTmJPReWVjwGm573gTPnzdCfrGaHVmYPiUDiE/w3nu8UbZVkRS1YD1oRtjXgestfZjfXhtygY6ONMKPLW2jD+u3E5VQ5CK2ibuOHcqx44vwlrLP1eXceHMcRRk+1n+fgWXPPhazPv+93UnM6ogK/oFOqyFyk2dW/Hl68CGu2+bkQelH+3cVZMT/YpPIpIaBiTQjTHDgW3An621V0RZ31tSFxQUFJCqgd5XD7y0mbv+tT6mbb94/HjOPWY0WyvrWfFBJe+WVTOmMIsff/4YCrI7hH1TLez6b3vA73gj+nBJgKGTYezM9pb8iCM0bYFICkl4oBun8/cR4AxgurW2W0Ip0HtW29RCeXUjI/KzWLczwAX3x/0VBKdNGU5+tp8b5k7pPl2BtbB/i9NNs+MNp6tmz9ruk44B+HNg9LHOcMnSGc4QysJx4NXl+0SS0UAE+o+AG4DLrbWL+1KoVO9yGQjBUJjP/XoFqz6Mr07+fd3JHDqyl2GNzfVQtqpzV01t92uxAmC8TqgPnQRDJ0buI7fC8eDPiqt8IpI4CQ10Y8xdwHeBa6y19/a1UAr0A9tb08SMu54DoKQgix985mgyfB6efXcPDy7f0m37b8z5CNfPPTT2A1gLgR2w4/X2L1zL3oFwsJcXGigodc527Rj0reGveeNFBlTCAt0YcztwC3CjtfaH/SmUAr3vNpXXcOeT63lh495u6zJ8Hppbwlx9ymSmjytk9mEj8HtjHK8eCkLgQ9i3xZnCoO1+M+zfCqGmXnfBkJGdA771cdFEDasUSYCEBLox5jZgAXCLtfbO/hZKgd5/wVCYn//nfe59ftMBt7v5rMO580nna47Hvnoix4ztQ7CGw1Czqz3g225bnftgDGPzs4dGb9UPneTMUKnpDkR6lYhx6DcAPwL+CdzVZXWTtfbteAulQE+ceIdFdvTObZ+gINtPQ3MIY8DnMfhibdG3shZqyzsH/f5I675yc/T5a7rKzG8P94KxkDsMcoc70yDkFkfuh6lLRw56iQj0F4BTeli9zVo7Id5CKdATy1rLa1v2sbemiUnDc/nrmzvYsb+e59aX9/7iLjwG3r/rTLyeBLSYrXUuEtIW9ls6B39Pwyt74st2gj2nOHI/7ADPhztz4KjlL2lEk3MdxPZUN/Kx7/+HQ0YM4VufnMIvX/iAd2IYRfPZj5byo89NG/gCNla3t+ZbW/TVO52gr6t07kPNfd+/N6N7Kz/q88gvgaxCzZGTbqx1PkMtTc73RKGmyOMelrUtb46yLOgM/7UhZ7K9cKj9cdf7bsvCnV/busyGnOXdloU6bN++rPCWTQQawwp06ey4O5+lovbAYTm+OIeWkGVnVQOj8rPYXd0Ydbv/OXUyN35yCis37yM/28eRowsSU0hroammc8DXVXS4r+zwPLI+2gyWsTLe9tZ+xhDwZztj9Tvdd3ickdt9WU/bezPS+6+FUEsk/JqciedCzd3DMNQalF3X9bRtMMagPcD6/jQIklDhwmoCTSjQpbtw2GIMbROFNbeEOeNnL/HB3v5NQPalj4/ntk8fmZhum3g113UP+QM9b64ZnHIZb/RfCgda5s1wpnWwkRZa2+Ow88uu0/Notw7bhLu+PpZ9WOe4nUK4Y9B2COlo00+kEuMBb6ZT574M57Evw3nuzQBfZJ3H6/wsPV7w+CKPPe3LTGR5t2Wt23s6L2tbF9uywlmXE6ipV6BLbOqaWjjytmcSsq9l3zyVicNyaQyG2F/fjMFgsZQUZCdk/wkRbHRa+h1b/fWVzi+GYEPk1vq4vsOy+u7LmuuIOjumtPP42kOyLSj9kTD1twdn2/qM7kHbtl200O24LDNKOGdG30+KTIGhPnTpk3DY8tTa3Xxk5BCGDclka2UdR48pINAQJNAQxAIZXg8+r+GHT2/kkbd3UpDtJ9DQ28lJMK20gL//zwnxj6hJdq39tZ1+GXQN/p5+KUTZLhR0WmbGRFppnh5u5gDrOq7vaR+9vL7HoI0Slm3b+rsEcUZ7C1f6TIEug6qpJcT1D7/Dk2vKYtr+iJJ8Lpw5ls/PGMvb26vI8Hk4dlzRAJdSJDUp0MUVT6/dza9e2MQ7O9rHoR9dWsDqHTGMSwc++H6Chk6KpBEFuiSdX76wiXue3tjrdu/deQYZvjTrlhHpBwW6JLVdVQ1c+5dVvL51X7d1p00Zzu8un+lCqUSSkwJdUoK1FmMM1lpm/WAZO6sa2tbd89mjObq0gMNG5Xd6TThs2VJZx6Rhud2v0SqShhToknKCoTBn37ucjXuijxGfVlrQqW8e4NWbZjO60BkOWV7dyHPryzlj6ig8HkNTMMSm8lrKa5ooyPbz9Nrd5GX5+NTUUXx0fJF+GUjKUKBLStpX18wVS97g7e3xfUaOHVfIf+N8zdDcDPbVNfPjz02jIRhi0fItfOfMw5l7xMi49iMy0BToktKstfx3exX3PL2B17Z072cfKBleD8u/fRoj8nWFJkkeCnRJO6397a3W7gxw9s+Xd9rm88eVsvTNHZ2W/eKiY8nweZg+rpBr/vI2r2yqPOBxTp0ynN9dNkNdMpI0FOhyUHBa8vvxGMPhJflk+eM7I7Gytonfr9xGVX2Q/Gw/9/7nfQCuO/1Qrjn9IwNRZJG4KdBF4hQKW2b/+AW2VbbP3PjE12ZxVGmCZpEU6aPeAl1nbYh04fUY/vWNk5g5cWjbsk/ft5z7nn+fUFiTb0nyUgtdpAfBUJjbHl/Hn17b3rYsP8vHH+Z/jKNL2xtI1lre2raf5lCYvTVNfLC3jlA4zLxjSynM9rO+rIaPTy7ucSqDdbsCZPo8ZPm95Gf7yc/yD/h7k9SkLheRfvrXmjK+8sf/9ns/p00Zztpd1RTnZvDhvnrqmkMH3P6yEyZwy9lHaE4baaNAF0kAay2/e2Urt//z3UE97vxZE7n57CMG9ZiSvBToIglUVd/MgsfX8diqXYAzXr05FOau86dy2Kg8RuZnMaYwm+376vndK1tZ/OpWpo7JZ+3O6qj7mzomnwyvhyNG5zMqP4v39tSypaKONTvbz4S9e95RXDhz3KC8P0luCnSRFBSoDzLt9n+3Pf/n12cxdYxG2RzsNMpFJAUV5Ph5/X/ntD3/3K9X8NjbO3GjASapQ4EukqRG5GXx5DdmMbogi4ZgiGsfXsV5v3yV17fsw1pLWEMopQt1uYgkuZ1VDdywdBUrN0efyyYnw0tOho9Mn4fPHDuGM48uoaE5xOtb9lFe08TrW/YxIi+Tw0ry2Li7lr21TRxTWkBOpo8vnziR4XmZg/yOpK/Uhy6SBqy1LNtYzk+fez/my/jFq6Qgi4raJkJhS7bfS2FOBiUFWdzwiSl8fHLxgBxT4qNAF0kj1lo2ldfy7Po9bN5bR0lBFiPzs9hSUccrmyrYsLv7HPLGgLVwyIghZPk97A40MjQ3g/f21MZ0zGy/l4f/3/GdTqYSdyjQRQ4idU0trNtVjddjmDw8l7wsf48nJu2taWLF5kpWfFDBW9v2M3VMAaMLsmkIhphQnEN9c6oQSt8AAAmESURBVIgHXt5MRW0zuRleFpxzJJ/9aKlmn3SRAl1E+mxTeS0X/3Yle6qbADiiJJ8zpo7i7GmjmTgs1+XSHXwU6CLSL/vrmrnpkdU8s25Pp+X5WT4OL8lnTFE2U0cXMCI/k5rGlrbrvxrAo2kLEkqBLiL9Zq1lxQeVPLmmjKfW7mZfXXNMrxs2JIOhuRkUZmeQn+2jJWzZvq+evEwfZYFGfB5DXXOIQEOQUflZfGTkEACaWsIMG5LBsCGZtIQtE4pzuPSECWT64pvnPt0o0EUkoYKhMG9u3c/6smr21TWzYXc17+wIsK+umSyfp9dJx/pq3FAn1E88pJhDR+QdlK3/hAS6MWYI8H3gc0AhsA643Vr7eF8KpUAXST+tWbK1sp5N5bXsr2+muiEIQE1jC9WNQYKhMIXZGWT6PBTlZtDcEgageEgG5dVNVNY1E7aWUNhS3xxid6CB6sYW/rt9Px2jamR+JidMHkZRTgbFQzIYNiSDCcW55Gf7I0Mu/eRk+GgIhmhoDuH1GLL8znmUuwONZPg8FGZnkJPpxecxWAsWkn5my0QF+rPAscCNwBbgMuBi4NPW2n/FWygFuojE44O9tfz6hQ948b29lNc0JXTfXo8hbC3WQpbfQ16Wn/wsH/nZfudsXGMItoTJ8nsYkZfF4SX5TBtbwNDcDDJ8HjK8Hufe5yHT6yXD58EYyPR5Ej4iqN+Bbow5E3gSmGetfTSyzAAvA8XW2sPjLZQCXUT6wlrL5oo6nlm3mw1lNW1/BeysaqSiNrFB31/GgN/rBL7fa/AYg8dj8BjwGoMxBm/kees6cC6B2BIOEwo5F0P3ew2ZPi/GwPPfPYtgQ22Pge6LoVznAwHgH60LrLXWGLMEuN8Yc4S1dnAniRaRg5IxhsnDh/CVUw/ptq65JUxNY5DGljD765ppCIbI9nvJ8ntpaA7REg7T3BKmdGgO1lr21wVpbAlR3xzC73ECta6phZrGFqrqm6lqCJLp82Kx+DyGxmCYXVUNrPqwivf21NDbVDrWOmVq7VZKhJZeDhpLoE8F3rXWdi3V6o7rO64wxvTW9C4IBAIUFurMMxFJXbbtn9Y7G2VZ+8Y9x3HXNabL2sjem+oA8nvaSyyBXgy8F2X5vg7r+yQQCAzMpBTpqXUybNVZ7FRn8VOdxW8w6ywf6LHJH0ugw4F+sURZ11P/TqvWFnxv20k71Vn8VGfxU53FL5nqLJb50CuJ3gofGrmPPqeniIgMqlgCfR1wuDGm67ZHRe7XJrZIIiLSF7EE+qM4JxN9usvyLwEbNcJFRCQ5xNKH/i9gGfCgMaYY58SiS4FZwLkDWDYREYlDr4EeGXN+Hs6p/9/Haa2/i3Oi0RMDXD4REYmRa5NzQXJ8K5wqVGfxU53FT3UWv2Sqs1j60EVEJAW40kIXEZHEUwtdRCRNKNBFRNKEAl1EJE0MaqAbY4YYY+41xpQZYxqMMW8aY84ZzDIMNmNMqTHmZ8aY5caYWmOMNcac2sO2Fxlj3jHGNBpjdhhjFhpjsqJsN9IYs8QYU2GMqTPGvGyMOaE/+0wWxpg5xpjFxpiNxpj6SJkfMcYcFWXbucaYlZHPUrkx5jfGmG4jDeL53MW6z2RijDnBGPOMMWZn5Oe81xjzvDHmjCjbqs56YIxZEPn/uSrKutSoN2vtoN2AZ3HmhrkCmA08BISAMwezHIP8nk8FyoGnceaUt8CpUba7JLLul8BpwFeAGuAvXbbLwpluYStwIfAJnJO/GoDpfdlnMt2AvwLPA1cDpwCfB94EGoHju9RrMLL96ThnLpcBrwCevnzu4tlnMt2As4B7gS9E3sP5wFORn/0XVGcx1eGRkf9Du4FVfX2PbtfbYFbYmZEP2PkdlhlgObDe7R/oAL5vT4fH5xEl0AFv5If5jy7Lr4xs/7EOy74SWXZsh2WZwGbgqb7sM5luwIgoywqB/cDfOyx7HXi7S/3Ojby3C/ryuYt1n6lwwzlp8EPgedVZr3XlAVYCPwdeoHugp0y9DWalPQBU0f03WmvAHOH2D3YQ6qCnQD8xsnxel+U5QDNwT4dlzwKro+z7LqAFyIt3n6lwi/wHeC3yeEzkvV0fZbsdwNJ4P3fx7DNVbjh/yT2jOuu1nm6IlDe/a6CnWr0NZh96LFc+Oli1vvdOM1daa+uBD+hcN1O7bhexGqdVfniH7WLdZ1Izxgyn8/uO+t4i1tC9vmL53MWzz6RkjPEYY3zGmNHGmO8BhwL/F1mtOovCGDMJuB34mrW2OsomKVVvgxnoxUSfO73fVz5KA63vvaf6Ke6ybSz1GM8+k5YxxgD343xWfxRZrPqKbilOv+xO4Frg89bapyPrVGddRD5bD+D8FfNYD5ulVL0N9rDFuK58dBDqqQ66Lo+nHmPdZ7L6IU5X1dXW2vVd1qm+OrsRmAmcg/NF+VJjzIVdtlGdtbsSOA74egzbpkS9DWag68pHPauM3PdUP/u6bBtLPcazz6RkjLkLp3/zGmvt4g6rVF9RWGs3W2vfsNY+Ya29EHgG+EXk4jSqsw6MMcOAe4C7gTpjTGFkyKAP8EaeZ5Fi9TaYga4rH/VsXeS+U9+ZMSYHmEznulnXdbuIo3CGR23owz6TjjHmduC7wI3W2nu7rI763iKOont9xfK5i2efqeJ1oAgYjuqsq1KcizvfjTOCqvV2Is772Q8sINXqbRC/ST4L50+Jc7ssfwnY4PY33YNUBz2NcvHhDDF8tMvyKyLbdxx//dXIsmM6LMvA+aLz6b7sM9luwG2RMt58gG3eAN6i87CvOXQfex3z5y7WfabCDWe43H9wgsmnOutWP0NwxoJ3va0CNkUeT0q1ehvsD9jzQAXwZZwTXRYDYeDTbv+AB/i9fzZy+0HkB3Zb5PkZHba5NLLuvsiH6X+AauCvXfaVhXOBkc3ABThjV/+Jc1LER7tsG9M+k+mG08VigSeA47vcpnfYbjbOMM2HI/8RvgjswhlP7O3L5y7WfSbbDfgjzsVnPoNzMtYXaD+x6Guqs7jq8gW6j0NPmXob7MrKj4TLbpwz//4LnOf2D3EQ3rft4ba1y3aX4AxbasIZqXAPkB1lf6OA3+P0tdXjnLgwq4djx7TPZLlF/kPFWl+fAl6LfJb24oxYKOrP5y7WfSbTDfgasAKnb7Ylcv9M1xBRncX8+VsVZXlK1JvmQxcRSROabVFEJE0o0EVE0oQCXUQkTSjQRUTShAJdRCRNKNBFRNKEAl1EJE0o0EVE0oQCXUQkTfx/HIhEjmLP2AwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_CEL.fit_one_cycle(10, 5e-4, div_factor=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-28T08:27:13.063544Z",
     "start_time": "2020-06-28T08:24:09.171389Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> \u001b[0;32m/home/skumar/miniconda3/envs/ImageCaption/lib/python3.7/site-packages/nltk/translate/bleu_score.py\u001b[0m(324)\u001b[0;36mmodified_precision\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m    322 \u001b[0;31m    \u001b[0;32mfor\u001b[0m \u001b[0mreference\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mreferences\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    323 \u001b[0;31m        reference_counts = (\n",
      "\u001b[0m\u001b[0;32m--> 324 \u001b[0;31m            \u001b[0mCounter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mngrams\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreference\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreference\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0mn\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mCounter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    325 \u001b[0;31m        )\n",
      "\u001b[0m\u001b[0;32m    326 \u001b[0;31m        \u001b[0;32mfor\u001b[0m \u001b[0mngram\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcounts\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\n",
      "ipdb> type(references)\n",
      "<class 'list'>\n",
      "ipdb> len(references)\n",
      "26\n",
      "ipdb> references\n",
      "[11, 16, 10, 1954, 17, 41, 19, 10, 49, 662, 27, 3476, 663, 12, 152, 13, 538, 13, 10, 476, 510, 12, 578, 2652, 94, 9]\n",
      "ipdb> exit\n"
     ]
    }
   ],
   "source": [
    "%debug"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T17:55:57.169355Z",
     "start_time": "2020-06-22T17:55:55.635574Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "learn_CEL.export()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-01T17:11:53.954568Z",
     "start_time": "2020-07-01T17:11:53.930671Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def get_predictions(learn, ds_type=DatasetType.Valid):\n",
    "    learn.model.eval()\n",
    "    inputs, targets, outputs = [],[],[]\n",
    "    with torch.no_grad():\n",
    "        for xb,yb in progress_bar(learn.dl(ds_type)):\n",
    "            out = learn.model(*xb)[0]\n",
    "            for x,y,z in zip(xb[0],yb,out):\n",
    "                inputs.append(learn.data.train_ds.x.reconstruct(x.cpu()))\n",
    "                targets.append(learn.data.train_ds.y.reconstruct(y.cpu()))\n",
    "                outputs.append(learn.data.train_ds.y.reconstruct(z.cpu().argmax(1)))\n",
    "    return inputs, targets, outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-01T17:12:04.314568Z",
     "start_time": "2020-07-01T17:11:54.239403Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='109' class='' max='109', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [109/109 00:09<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "inputs, targets, outputs = get_predictions(learn_CEL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-01T17:12:06.535110Z",
     "start_time": "2020-07-01T17:12:06.514608Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Text si on parle des gens en xxunk , xxunk que la dynamique de travail et xxunk est différente de celle d’une installation permanente ?,\n",
       " Text when it comes to the people on tour , i xxunk that the xxunk dynamic is different from what goes on in a permanent production ?,\n",
       " Text whether the comes to the people , xxunk , are and and the work work and different from a a ? a a food resident ?)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs[500],targets[500],outputs[500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-01T17:12:19.198213Z",
     "start_time": "2020-07-01T17:12:19.178174Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Text quels sont les prix de vente en gros et au détail que vous prévoyez demander pour votre ( vos ) xxunk ) ou produit xxunk contenant notre matériel ?,\n",
       " Text what will be your anticipated xxunk and retail price for each of your xxunk ) or any work derived from using our material ?,\n",
       " Text what are xxunk the price price and selling about xxunk your of your product and that product material ? from your for product ?)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs[22],targets[22],outputs[22]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-24T15:53:30.543057Z",
     "start_time": "2020-05-24T15:53:30.524197Z"
    }
   },
   "source": [
    "### Fastai - with LabelSmoothingLoss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-02T08:23:14.183400Z",
     "start_time": "2020-07-02T08:23:14.144641Z"
    }
   },
   "outputs": [],
   "source": [
    "class SmoothLabelCritierion(nn.Module):\n",
    "    \"\"\"\n",
    "    TODO:\n",
    "    1. Add label smoothing\n",
    "    2. Calculate loss\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, label_smoothing=0.0):\n",
    "        super(SmoothLabelCritierion, self).__init__()\n",
    "        self.label_smoothing = label_smoothing\n",
    "        self.LogSoftmax = nn.LogSoftmax()\n",
    "\n",
    "        # When label smoothing is turned on, KL-divergence is minimized\n",
    "        # If label smoothing value is set to zero, the loss\n",
    "        # is equivalent to NLLLoss or CrossEntropyLoss.\n",
    "        if label_smoothing > 0:\n",
    "            self.criterion = nn.KLDivLoss(reduction='batchmean')\n",
    "        else:\n",
    "            self.criterion = nn.NLLLoss()\n",
    "        self.confidence = 1.0 - label_smoothing\n",
    "\n",
    "    def _smooth_label(self, num_tokens):\n",
    "\n",
    "        one_hot = torch.randn(1, num_tokens)\n",
    "        one_hot.fill_(self.label_smoothing / (num_tokens - 1))\n",
    "        return one_hot\n",
    "\n",
    "    def _bottle(self, v):\n",
    "        return v.view(-1, v.size(2))\n",
    "\n",
    "    def forward(self, dec_outs, labels):\n",
    "        # Map the output to (0, 1)\n",
    "        dec_outs = dec_outs[0]\n",
    "        scores = self.LogSoftmax(dec_outs)\n",
    "        # n_class\n",
    "        num_tokens = scores.size(-1)\n",
    "\n",
    "        gtruth = labels.view(-1)\n",
    "        if self.confidence < 1:\n",
    "            tdata = gtruth.detach()\n",
    "            one_hot = self._smooth_label(num_tokens)\n",
    "            if labels.is_cuda:\n",
    "                one_hot = one_hot.cuda()\n",
    "            tmp_ = one_hot.repeat(gtruth.size(0), 1)\n",
    "            tmp_.scatter_(1, tdata.unsqueeze(1), self.confidence)\n",
    "            gtruth = tmp_.detach()\n",
    "        loss = self.criterion(scores.view(-1, scores.size(-1)), gtruth)\n",
    "        return loss\n",
    "    \n",
    "    \n",
    "def LabelSmoothingCrossEntropy_func(input,targets):\n",
    "    x,y = input,targets\n",
    "    loss = SmoothLabelCritierion(label_smoothing=0.1).to(device)(x,y)\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-02T07:19:57.423865Z",
     "start_time": "2020-07-02T07:19:57.341394Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/skumar/miniconda3/envs/ImageCaption/lib/python3.7/site-packages/torch/nn/_reduction.py:43: UserWarning: size_average and reduce args will be deprecated, please use reduction='sum' instead.\n",
      "  warnings.warn(warning.format(ret))\n"
     ]
    }
   ],
   "source": [
    "opt_fn = partial(optim.Adam, betas=(0.9, 0.98))\n",
    "\n",
    "criterion = LabelSmoothing(len(en_itos), 1, 0.1).to(device)\n",
    "\n",
    "learn_LSL = Learner(databunch,model,loss_func=LabelSmoothingCrossEntropy_func,wd=0.1, metrics=[seq2seq_acc, CorpusBLEU(len(en_itos))],callback_fns=[ShowGraph]) #,TeacherForcingTurnOff,TeacherForcingCallbacklearn = "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-02T04:09:37.923592Z",
     "start_time": "2020-07-02T04:09:18.688670Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='1', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/1 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>seq2seq_acc</th>\n",
       "      <th>bleu</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='99' class='' max='435', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      22.76% [99/435 00:17<01:00 2.8333]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/skumar/miniconda3/envs/ImageCaption/lib/python3.7/site-packages/ipykernel_launcher.py:34: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n",
      "Min numerical gradient: 2.51E-03\n",
      "Min loss divided by 10: 3.02E-04\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAAEZCAYAAADYGFGeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3wVVfrH8c+TTgIkQOi9SRcpCggKir0gKuqKotix7OKqa1nrurqui/uzroVFRbFgQ7AXRFSkiEiX3ntPCIH08/tjbtgY00ky9ybf9+s1r0tmzsw8M8B9cs6cOcecc4iIiPgtzO8AREREQAlJRESChBKSiIgEBSUkEREJCkpIIiISFCL8DiAUmVkWXjLf73csIiIhpDaQ45wrMPeYun2XnpnlABYfH+93KCIiISM5ORnAOecKbJ1TDals9sfHx8cnJSX5HYeISMhISEggOTm50JYlPUMSEZGgoIQkIiJBQQlJRESCghKSiIgEBSUkEREJCkpIIiISFNTtu5Kt3JHC4s3JOMA5hwPI9ypYTFQ49eKiqBMbRWLNKOrXisbMfIhWRKTyKCFVsmnLd/LPz5eXap+GtaPp26YefdvUY0C7RJrXja2g6ERE/KOEVMlqRkfQOD4Gg8O1HjNvAXAODmZks+9gBrmDaOzYn86UBVuZsmArYQaX9WnJHad1ID420p+LEBGpABo6qAzMLKmiR2rIznHsP5TJlqRDzF2/lzlr9zJn3R72HcwEoG5cFHed0YEzujQmNSOLgxnZOOdoU78m4WFq3hOR4BMYqSHZOZdQ0HYlpDKojIRUkMzsHF6buZ6npq7iQHpWgWUSYiM5oX19Bh5Vn0Ed6pNYM7pSYxQRKYwSUgXwKyHl2rk/jX98tozJC7YWWS46IoyrB7TmxkFtqR2j5j0R8ZcSUgXwOyHl2rT3IPvTMomLiiA2OpxDGdnMWL2b6St2MXP1blIzsgGoExvJnwa357I+LYmKUE9/EfGHElIFCJaEVJTU9Cz++8Naxn6/loOBxNS5cW2eufQY2jWo5XN0IlIdKSFVgFBISLl2pqTx1NRVTPxpIzkOYiLDuP+czgw/roXebRKRSqWEVAFCKSHlmrt+L7dOXMCWpEMAnNq5Ifed3YmW9eJ8jkxEqgslpAoQigkJIPlQJvd+uJhPFm0DIMxgSPcm3HRSO45qqGY8EalYSkgVIFQTEnjDFU1esIUnvlx5uLYEcFa3Rtx5ekdaJarGJCIVQwmpAoRyQsqVkZXD5AVbeGH6GtbtTgUgMtwY0bcVfxrcjoTYKJ8jFJGqRgmpAlSFhJQrO8fx8cKtjPlyxeEaU3yNSP59UXdO6dzQ5+hEpCopLiHppZRqLjzMGNqjKd/cPpC7zuhIregIkg9lcvNbvzBvwz6/wxORakQJSQCIiQznxkFt+eaOgbROjCM9K4frXv+Z9YHmPBGRiqaEJL/RoFYMr448lrpxUexNzWDkqz+xNzXD77BEpBpQQpLfaZUYx3+v6E1URBjr9xzkutd/JjkwyriISEVRQpIC9WpZh6cuOQYzmLdhH2c984OeKYlIhVJCkkKd1a0x/76oOzGRYWxJOsTFL83iP9+uJidHPTNFpPwpIUmRLujZjI9vGUDHRrXIznGM+XIFV7zyE7tS0v0OTUSqGCUkKVb7hrWYfHN/Lu/bAoAZq3dz1jM/MHP1bp8jE5GqRAlJSiQmMpxHhnbj+ct6Uis6gl0p6Vz28hye/Hol2WrCE5FyoIQkpXJWt8Z88qcBdGsaj3Pw9Der+Mt7C/VcSUSOmBKSlFrLenG8f2O/w014k+Zv4b4pS9AwVCJyJJSQpEyiI8L5+3lduap/KwDemrORRz5dpqQkImWmhCRlZmY8cE5nLj2uOQAvz1jHmC9XqPlORMpECUmOiJnxyNBunN+jKQDPT1/DyPFz2bk/zefIRCTUKCHJEQsPM8YMO5o/HOvVlL5fuYvTn/qeL5Zs9zkyEQklmg+pDKrSfEjl7bPF27hn0mKSD3lj3w3p3oS7zuxI04QaPkcmIn7TBH0VQAmpaNuSD3HHewv5cfUeAKIjwrjuhDaMGtSWmtERPkcnIn5RQqoASkjFy8lxfPDLZsZ8uYKdgWGGEmtGc8+ZHbmgZ1PMzOcIRaSyKSFVACWkkktNz+Kl79cy9vs1pGXmANC7ZR0ePq8rnZvU9jk6EalMSkgVQAmp9LYmHeLRz5bx6aJtAIQZ3HxSO24/rYPPkYlIZSkuIamXnVSKJgk1+M/wnrx5bR/aNahJjoNnp61WTzwROUwJSSpV/3aJfPanEzihfSIA909ZQtJBTZEuIkpI4oOoiDAeu6AbcVHh7EpJ5++fLPM7JBEJAr4mJDMbbGbjzWyFmR00s81mNsnMupVw/6vMbLGZpZvZDjMba2b1CigXaWZ/M7MNgbJLzeya8r8iKalmdWK5+6xOAHzwy2a+XbHT54hExG9+15BGAS2AJ4EzgdsCP881s75F7WhmtwOvALOBIcC9gc+vzSwyX/EXgL8ATwGnA58D48xsVPldipTWZce1oE/rugD8ddJiUtIyfY5IRPzkay87M2vgnNuZb10CsA6Y5py7sJD9YoCdgTJD86wfCEwHrnfO/TewrguwBLjNOfdknrJv4iXBJs65Ug28pl525Wf97lTOePp70jJzOLd7E575wzF6R0mkigrqXnb5k1FgXRKwCmhWxK5dgFrAJ/n2/Q7YDwzLs3oo4IAJ+Y4xHqgDnFzauKX8tEqM4+4zOgLw8cKtPP3NKp8jEhG/+N1k9ztmVh/oilerKUxU4DO9gG3pgf1zdQW2O+d25yu3KM928dGVx7fikt7ewKxPTV3FlAVbfI5IRPwQVAnJvLaasXhxPVFE0RVADnB8vv2PAuoDiXlW1wP2FnCMvXm2548jqagFiC/xRUmxzIy/D+3K8W29v4q/vLeIn9cX9FcmIlVZUCUkYAxeE9so51yhfYGdc3uBN4Grzew6M6trZt2BN4BsvGT1m10KOkwR26SSRUWE8cJlvWhTP46M7ByunzCP7cmaU0mkOgmahGRmjwK3A6Odc+NLsMuNwLvAi8Ae4BfgV+BTYGuecnsooBaUZ93vfhV3ziUUtQDJJb0uKbn42EheHXksdWIj2ZuawQNTimq1FZGqJigSkpk9DPwVuNM590xJ9nHOpTrnRuA1z3UHGjrnRgIdgR/yFF0KNCrg/aTcd530rRdEWtaL46EhXQD46tcdfLFkm88RiUhl8T0hmdmDwP3A/c65MaXd3zm3zzm3yDm328zOB9oDz+cpMhkw4PJ8u14JJAHfli1yqShDujfhpA71Abh/ytLDk/2JSNXm90gNtwMP4XXfnmpmffMsPfKUm25mLt++F5vZH83sFDM718z+D3gPeNg591NuOefcErwu3o+Z2a1mNtDMxuAlqPucc4cq/EKlVMyMR87vRmxgaKF/fr7c75BEpBL4PX3nuYHPcwJLXhuAVkXsmw1cA7QL/LwIGO6ce7eAsjcAm/FGgmgIrCXPy7MSfJom1OCO0zrw8Ce/8vZPGznvmCb0bVPQo0ARqSo0H1IZaKSGypGd47jghZks3JRE2/pxfHnriUSE+97KLCJlFNQjNYgUJTzMeOz8bpjBml2pTJy7ye+QRKQCKSFJUOvcpDbDenqjSD01dSUH0rN8jkhEKooSkgS920/rQExkGLsPZDD2+7V+hyMiFUQJSYJeo/gYrh3QBoD/fr+WHfs1goNIVaSEJCHhhoFtqBcXxaHMbJ6autLvcESkAighSUioFRPJ6FPaA/DO3E2s2J7ic0QiUt6UkCRkXHpcC1onxpHj4Oa3ftEMsyJVjBKShIzI8DDGDDuayHBj9c4D/PmdheTk6D06kapCCUlCSu9WdXn4PG9OxanLduh5kkgVooQkIefS41pwRb+WADwzbTWfLdaI4CJVgRKShKT7z+lMn9Z1Abj93YWs253qc0QicqSUkCQkRYaH8fxlPWkcH8OhzGxuf3cBWdn5JwoWkVCihCQhq17NaMYM6w7ALxuTeEmjOIiENCUkCWkD2icy8vhWgDfW3dKtml1eJFQpIUnIu+uMjrRJjCMz23HbOwtJz8r2OyQRKQMlJAl5NaLC+ffF3QkzWLEjhYc//hXN8yUSepSQpEro0aIOt5zkTR785pyN3Dt5iV6aFQkxSkhSZYw+5Sj+cGxzAN6as5E73luonnciIUQJSaqM8DDjsQu6He7kMGn+FkZPXECmkpJISFBCkirFzHjw3M7cNKgtAJ8u3sYrM9b5HJWIlIQSklQ5ZsadZ3Tksj4tAHju29XsS83wOSoRKY4SklRZd5zWgVoxEaSkZfHstNV+hyMixVBCkiqrTlwUNwd63k2YvZ4NezTenUgwU0KSKm3k8a1omlCDzGzHv75c4Xc4IlIEJSSp0mIiw7n9tKMA+HTRNuZv3OdzRCJSGCUkqfKGHtOULk1qA/DIp8vI1guzIkFJCUmqvLAw496zOgEwb8M+zTIrEqSUkKRaOL5dIlf1bwXAs9NWM235Dn8DEpHfUUKSauOeMzvRq2UdAG6duIBNew/6HJGI5KWEJNVGVEQY/xnek3pxUexPy2LUG/NIy9RUFSLBQglJqpVG8TE8e2kPwgyWbt3P/32t50kiwUIJSaqd49sl8seT2wMwfuZ6tiUf8jkiEQElJKmmbhjYhsSa0WRk5WhYIZEgoYQk1VJsVAS3nOSNCP7u3E0aVkgkCCghSbV1aZ8WNE2oQVaO46mpq/wOR6TaK3VCMrN2ZnZGvnV9zOxjM/vRzK4vv/BEKk50RDijB3vPkiYv2MLKHSk+RyRSvZWlhvQ4cFfuD2aWCHwOnA50BV4ws6HlE55IxbqgZ1PaJMbhHPzfV+pxJ+KnsiSk3sDUPD9fCtQGegL1gTnA6CMPTaTiRYSHceup3uCrXyzdzrwNe32OSKT6KktCqg9szfPzGcCPzrklzrkMYCLQuTyCE6kM53RrTLem8QDcM2kxGVk5PkckUj2VJSGlAgkAZhYODAC+z7P9EF6NSSQkhIUZj13QjfAwY+WOA7z43Rq/QxKplsqSkJYCI8ysHnAdUBP4Os/2lsCucohNpNJ0bRrPNQNaA/DctNWs3nnA54hEqp+yJKQxwNHATuA/wHzghzzbTwN+OfLQRCrXrae0p3ndGmRk5/DXSYvJ0bxJIpWq1AnJOfcpcDLwFPA34DTnnAMI1Jo2A+OLO46ZDTaz8Wa2wswOmtlmM5tkZt1KEoeZXWhmM81sX2CZZWYXF1DOFbKMKsVlSzUQGxXBo0O9f34/rd/L23M3+hyRSPVigVxS+Sc2ew+oB7wLLAMaAnfidR0f5JybXcS+V+IlvQ+AlwOrrwEuBK5xzr2Sp6wD3sFLoHmtdc7tLGPsSfHx8fFJSUll2V2C3J/fWcCH87cQHRHG61cfR5829fwOSaRKSEhIIDk5Odk5l1DQ9nJJSGYWAZwH1AU+ds5tL8E+DfInBDNLANYB05xzFxax73SgFdDGOZcTWBcGrAXWO+cG5SnrgKedc7eW8rKKil0JqQrbl5rBhS/MZO3uVGrFRPDO9f3o3ET9dESOVHEJqSwjNfzLzObm+dnw3kt6F3gJWGxmbYs7TkG1E+dcErAKaFbM7pnAgdxkFNg3BzgApJfkOkQKUycuiteuPo6GtaNJScviyld/YuMeTeYnUtHK0qnhDH7bieFc4ES8zg7DA+vuLkswZlYfr8luSTFFnwM6mdm9ZpZoZvXN7F6gA/BkAeWvMLNDZpZmZnMKetYkklfzurG8fnUfasdEsCslnRGvzGFXin7XEalIZUlIzfFqMbnOBdY55+52zk0EXgQGl/aggZrW2EBMTxRV1jk3BRgC3IHXxXwncA9wkXPui3zF3wRuwev9dwXee1LvmFmho0mYWVJRCxBf2uuT0NOhUS1evepYYiLD2LDnINe+NpdDGZphVqSilCUhRQF5/1eexG+HEloLNC7DcccAQ4FRzrllRRU0s1OBt/A6K5wGnAl8CLxtZmfnLeucu9w595Zz7gfn3Lt4PQR/AB4xsxpliFOqkV4t6/Kf4T0JM1i4OZnb3l2g7uAiFaQsCWkT0BfAzLoAbYDv8mxvgPcsp8TM7FHgdmC0c258MWUNeA2v48Mo59zXzrkvnHMjgNnAs0XtH3jW9AbeC71dCymTUNQCJJfm+iS0De7UkAfO8UbD+nzJdh7/crnPEYlUTWVJSBOBK83sE+ATYD/wWZ7tPYASj71iZg8DfwXudM49U4JdGuLVwH4uYNvPQGsziynmGLnXrUHLpERG9m/NyONbAfDSd2t5+ye9oyRS3sqSkB7DeweoH+CAKwK94zCzeLxnO9+U5EBm9iBwP3C/c25MCc+/D0gDjitgW19gj3MurYhzhgGXASl4wyCJlMj953Tm5I4NvD9PXqKedyLlrCwjNaQ7565xztVzzrVxzn2UZ3MKXu3loeKOY2a3B8p9Akw1s755lh55yk0PvEt0+Px4HSeGmNk4MzvDzM42s3fwBnp9Ms++d5jZf83sUjMbZGZ/AL4NlLu7qMQlkl94mPHspT1oEh9DVo7jpe81CKtIeSrXKcydcznOuWTnXGYJip8b+DwHmJVv+bCYfe8ARuE1D04EJuAN6joC+EeeciuAjsAzeAPAvghkAUOcc8+X5JpE8oqLjuD6E9sA8N68zezcr99pRMpLmUZqMLM4vGF+zsfr1ABe77pJwBjnXGq5RRiENFJD9XYoI5sBj09jT2oGN5zYhnvO6uR3SCIhoSJGaqgL/IT37KcR3mjf8/E6GzwA/BQoI1Il1YgK5+rAVBVvzN5A0sEMnyMSqRrK0mT3MF4z2C1AY+fcCc65E4AmwM14oyU8VG4RigShEf1aUis6gtSMbF6bucHvcESqhLIkpCHAOOfc8865wy/IOueynXMvAK/gveAqUmXVjolkRL+WALw6cx2p6Vk+RyQS+sqSkBriNdEV5pdAGZEq7eoBrYmOCCPpYCZvzdF7SSJHqiwJaQde77bC9AiUEanSEmtGc+lxLQAY+8Na0jI1zp3IkShLQvoYuMbMbgi8ZAp4L5ya2fXA1cBHhe4tUoVcf2IbosLD2JWSrlqSyBEqS0J6AK+L9/PAVjP7zsy+A7YCLwS2PVh+IYoEryYJNbj4WG/6rhe/W6NaksgRKMtIDXuA3sA/gT3AsYFlN96wQr0DZUSqhRsHtSMy3NiZks5EjXEnUmZlGqnBObffOXevc66Lcy42sHR1zt0HDDezX8s5TpGg1TShBhf3bg7AC6oliZRZuQ4dFJCI9y6SSLVx00leLWnH/nTembvJ73BEQlJFJCSRaqdpQg2G9QrUkqarliRSFkpIIuXk5pPaEhFmbN+fph53ImWghCRSTprVieUPx3m1pGenrWJ/WkkGvReRXEpIIuVo9OCjiI0KZ9/BTF6crvmSREojoiSFzOy2UhyzfxljEQl59WtFc/2JbXhq6ipenrGOEf1a0ji+ht9hiZSbQxnZLNqcRJ829cr92CWaD8nMckp5XOecCy9bSMFP8yFJUVLTsxg4Zjq7D6Rzce9m/GtYd79DEjkiWdk5zFi9mykLtvLl0u1kZucw995TSIiNKtVxipsPqUQ1JOCkUp1VpBqLi47g1lPac9/kJbw/bzPXDGhDh0a1/A5LpFSccyzeksykX7bwyaKt7D7wv3m/IsONBZuSGNShQbmes0wzxlZ3qiFJcTKzczj9qe9ZuyuVkzs24JWRx/odkkiJbEs+xIfztzDply2s3nngN9uOa12X845pwtndGpe6dgTlV0MSkVKIDA/jztM7MuqNeUxbvpPl2/fTsVFtv8MSKVBaZjZfLt3O+/M2M2P1bvLWU9o3qMn5PZty3jFNaZpQsc9DlZBEKshpnRvSpn4ca3elMv7H9fzzwqP9DknksNwmuXd/3sSUBVtJSfvfJJP14qIYckwTLuzZjC5NamNmlRKTEpJIBQkLM0Ye34oHpizlw/lbuOuMjtSJK30zh0h5Sj6YyeQFW5g4dxPLtu0/vD4izBjcqQHDejVnUIf6RIZX/ltBSkgiFejCns0Y8+UKUtKyeHvuRm4a1M7vkKQacs7x84Z9vD1nI58u3kZ61v86Th/VsCYX927O+T2aUq9mtI9RKiGJVKi46Agu6d2ccTPWMWHWBq47oY0vv3lK9bQ/LZPJ87fw5uyNrNiRcnh9bFQ4Q7o34Q/HtaB7s/hKa5IrjhKSSAW7ol8rXv5xHduS0/hq6Q7OPrqx3yFJFffr1v1MmL2eyfO3cijPQL9HN4vn0uNacG73JtSMDr6v/+CLSKSKaVEvllM6NeTrX3fw6o/rlJCkQmRk5fDF0u28PnM9P2/Yd3h9jchwhvZowvDjWtKtWbyPERZPCUmkElzVvxVf/7qDnzfsY/Hm5KD/YpDQsftAOm/P2ciE2RvYmZJ+eH3b+nFc0a8V5/dsSu2YSB8jLDklJJFK0K9NPTo2qsXy7Sk8/sVyXr/6OMLCgqPdXkLTsm37eXnGOj5asJWMbK+TQpjBqZ0bcmW/VvRrWy9ong2VlBKSSCUwM2479SiunzCPGat389qs9VzVv7XfYUmIyclxTF+5k5dnrOPH1XsOr4+vEckfjmvOiL4taVYn1scIj4wSkkglOa1LIy7q1Yz35m3mn58vZ0C7RNo31Bh3Ury0zGymLNjC2O/XsmZX6uH17RrU5Or+rTm/R1NqRIX+eNZKSCKV6MEhXZi9bg+b9h5i9MQFTL65P1ER6gYuBUs+lMkbszfw6o/r2X3gf8+HTmifyNUDWjOwff0q1fSrwVXLQIOrypH4ef1eLn5pFjkORg1sy91ndvQ7JAkyO1PSeHnGOt6cvZED6d6QPhFhxpDuTbj2hDZ0bhKa4yJqcFWRINO7VV1uHNSW/3y7hpe+X8OQ7k1C9gtGytemvQd58bs1vDdvMxmB0RTiosIZ3qcFV/VvTZMKHtzUb0pIIj4YPfgovly6g9U7D/D89NU8N7yn3yGJj9bsOsDz365h8oItZOd4rVZ146K4un8rRvRtRXxsaHTbPlJKSCI+iIoI4+aT2vLndxby2eJtrNudSuvEOL/Dkkq2emcKz3yzmo8XbT085UPj+BhuOLENlxzbokp0VCgNJSQRn5x7dBP+/dVKNu87xEvfrdH0FNVIQYmoRd1YbhrUlgt6Nqu2HV2UkER8EhEexg0D23L/5CV88MtmRp/SnsbxVfsZQXW3bncqT09dyZSF/0tErerF8seT23PeMU2IqOYD7yohifjool7NeHrqKnYfSGfcD+u4/5zOfockFWDzvoM8880qPvjlf8+IlIh+TwlJxEcxkeFcM6A1j3+xnLfmbOTmk9pRV5P4VRm7D6Tz3LTVvDlnA5nZXiJqVqcGfxrcngt6NFUiykcJScRnl/dtwfPTV5OSlsX4H9dx22kd/A5JjlBKWib//X4t42as42CGN/1Dw9rR/PHk9lzcu3m1fUZUHCUkEZ/Vionkyn6teO7b1YybsY6Lejened3QHY+sOsvIyuGtORt4Ztpq9qZmAJAQG8lNg9pyRb9WxERWr15zpaWEJBIErjuxDe/N28SO/encM2kxE645LuRGaq7OnHN8tng7//pyORv2HAS8eYiuGdCa6we2CZnpH/ymhCQSBOJrRPLo0G5c+/rPzFi9m3d/3sQlx7bwOywpgfkb9/HIp8uYF5gULzzMuLh3c/58Snsa1I7xObrQ4ltDppkNNrPxZrbCzA6a2WYzm2Rm3Uq4/4VmNtPM9gWWWWZ2cSFl/2RmK80s3czWmNmdZqZGXAkqp3RuyJDuTQB45JNlbE9O8zkiKcqWpEOMnjif85+feTgZndKpIV/eegKPXdBNyagM/PxSHgW0AJ4EzgRuC/w818z6FrWjmV0JvA9sBYYHli3AO2Z2db6y9wXOMRE4HXgZeBT4R3lejEh5eGhIF+rFRZGSnsV9kxejwY+Dz6GMbJ78eiUnPzGdKQu2AtC5cW3euq4P467sTbsGmlKkrHwb7dvMGjjnduZblwCsA6Y55y4sYt/pQCugjXMuJ7AuDFgLrHfODQqsqwdsBsY650bn2f9R4E6gtXNucxli12jfUmE+XriVP749H4CnLjmGoT2a+hyRgPec6JNF23jss2VsDdRe69eK5s7TO3BBz2aEV6FpICpKcaN9+1ZDyp+MAuuSgFVAs2J2zwQO5CajwL45wAEgPU+5M4AY4LV8+4/He342pNSBi1Swc45uzGmdGwLwwJQlbEs+5HNEsnJHCsP/O4c/vj2frclpRIWHceOgtnx7xyAu6t1cyaicBNVzFDOrD3QFlhRT9Dmgk5nda2aJZlbfzO4FOuA1z+XqCjhgad6dnXOrgEOB7SJBxcz4xwXdqBcXxf60LP7y3iJyctR054cD6Vk8+umvnPX0D8xa600Zfmrnhnx924ncdUZHakarX1h5Cpq7aV4f17F4SfKJoso656aY2RDgDeCRwOpU4CLn3Bd5itYDDjrn0vMfA9gX2F5QLMW1xcUXs13kiCTWjOYfF3TjhgnzmLF6N6/PWs/I/q39DqvacM7x+ZLt/O3jpezY7319tKoXy0NDujCoQwOfo6u6giYhAWOAocBVzrllRRU0s1OBt4C3gQ+AcOAy4G0zG+ac+zRP8aJ+tdSvnRK0Tu/SiGG9mvH+vM089vlyBrSvT7sGNf0Oq8rbsCeVB6Ys5buVuwCIiQzjjye359oTWhMdoRdbK1JQJKRAJ4PbgdHOufHFlDW8Z0LTnHOj8mz6wsyaAc8CuQlpDxBnZtEF1JLqAHsLOkdhD9zyxJCEaklSCR44tzOz1uxhS9Ihbn93AZNu6q/nFRUkMzuHsd+v5ZlvVpEemK11cMcGPDSki0bOqCS+P0Mys4eBvwJ3OueeKcEuDYHGwM8FbPsZaG1muS8ALAUM6JLvnO2AGhT/rErEV7VjInniou4ALNyczBdLtvscUdW0YFMS5z47gzFfriA9K4fG8TG8NKIX467srWRUiXxNSGb2IHA/cL9zbkwJd9sHpAHHFbCtL7DHOZf7RuHneL3uRuQrdyWQBXxc6qBFKlm/tvU4NdDr7rlvV+vdpHJ0MCOLhz/+lfOf/5Hl21MIM7i6f2um3jaQ00WlkQcAABYxSURBVLs00vBNlcy3Jjszux14CPgEmJrvZdh059z8QLnpwEDnnAE459LN7EXgVjMbh/eCbDhwBTAAuC/3IM65PWb2GHC/mSUD3wL9gLuAp5xzmyr2KkXKxy0ntePrX3ewbNt+vl2xk5M7NvQ7pJA3c/Vu7pq0iE17vW71HRvV4p8XHs0xzYtssZcK5OczpHMDn+cElrw24L34Wpg7gOXA9cAwIAdYiVcTejNf2YeBZOBm4B680R0eBB4ve+gilat78wROaJ/ID6t28+y01ZzUoYF+ey+jlLRMHvvcm38KICo8jNGntOf6E9sQqfmJfOXbSA2hTCM1iB/mrN3DJWNnA/DWtX04vl2izxGFnpmrd/OX9xexJcmrFfVokcCYYUdruJ9KUtxIDUHRy05EitenTT2ObVWHuev38dy3q5WQSuFgRhaPf76c12ZtACA6Ioy/nN6Bq/q3Vq/FIKL6qUgIueXk9gDMXLPn8AjTUrRfNu7j7GdmHE5GPVok8NnoE7j2hDZKRkFGCUkkhJzYPpFuTb1X4B7/fDnZGlKoUFnZOTz59UouenEW63anEhlu3HlGB967oR9t6+sF42CkhCQSQsyMO07vAMBP6/cy7oe1PkcUnNbtTmXYi7N4+ptVZOc4OjaqxUe3DOCmQe2IUMeFoKW/GZEQM/Co+gzv480m+8RXK/h1636fIwoezjne+3kTZz/zAws2eZ2OrjuhNZNv7k+nxrV9jk6Ko4QkEoLuO7sTrRPjyMx2/PmdBaRlZvsdku/2p2UyeuIC/vL+Ig5mZNOodgxvXduHe8/uTEykxqALBUpIIiEoNiqC/7u4O+FhxoodKfz7qxV+h+Sr+Rv3cfYzP/DRQm8G19O7NOSLW09QT8QQo4QkEqJ6tKjDLSe1A2DcjHXMWLXb54gqn3OOcT+s5aIXZ7Fp7yGiI8J4ZGhXXry8FwmxUX6HJ6WkhCQSwm45uR3HNE/AObj1nfns3J9W/E5VRNLBDK57fR6PfLqMrBxH+wY1+eiWAVzet6VGsQhRSkgiISwyPIxnL+1B7ZgIdh/I4E8T51eLruALNyVx9jMzmLpsBwDDejVjyi396dBIIy6EMiUkkRDXvG4sYwJTVMxeu5env1nlc0QVxznHm3M2cNGLs9iSdIiYyDDGDDuaJy7qTmyUBp4JdUpIIlXA6V0acVX/VgA8O21VlXyedCgjm9vfW8i9Hy4hIzuHNolxfHTLAC7q3dzv0KScKCGJVBH3nNmJ7s3icQ7+/O4C9qdl+h1Sudm09yAXvDCTSb9sAeCsbo2Yckt/jmqoJrqqRAlJpIqIigjjueE9qREZzq6UdJ76umo03c1YtZtzn5vBsm37CQ8z7ju7E/8Z3pNaMZF+hyblTAlJpAppXjeWPw72uoK/Nms9y7eH7igOzjn++/1arnhlDkkHM6kbF8Ub1/Th2hPaqBddFaWEJFLFXDugDW0S48jOcTwweWlITnmelpnNbe8u5NHPlpHjoGvT2nz8xwH0a1vP79CkAikhiVQxURFhPDSkC+ANwDplwVafIyqdnSlp/GHsbD6c7z0vOr9HU94fdTxNE2r4HJlUNCUkkSroxKPqc2bXRgA8+tkyUkKkg8OSLcmc99yPLNiUhBncc2ZH/u/i7hqLrppQQhKpou47p/PhDg7/+iL4x7r7aul2LnpxFtuS04iLCmfcFb25YWBbPS+qRpSQRKqopgk1GH2KN8PshNkb+HbFTp8jKphzjpdnrOOGN+ZxKDObZnVqMOmm/gzu1NDv0KSSKSGJVGHXndCGY1vVAeDO9xexNzXD54h+Kys7hwc/WsrfP/kV57zpxSffrCGAqislJJEqLDzM+L+Lj6FmdAS7UtK5+4NFQdPr7mBGFtdPmMfrszYAcHa3xrx9XV8Sa0b7HJn4RQlJpIprXjeWvwV63X316w7e+3mzzxHB7gPpXDp2NtOWe82INw5qy7OX9lDnhWpOCUmkGrigZ1PO7tYYgIc+XurrNBUb9qQy7IWZLNycTJjBI0O7ctcZHQkLU+eF6k4JSaQaMDMePb8rdWIjOZiRzQeBMeEq2+LNyVzw/EzW7zlIdEQYL17ei8v7tvQlFgk+Skgi1URCbBTnHdMUgEm/bK70Z0kz1+zmD2NnsSc1gzqxkbx1XV9O69KoUmOQ4KaEJFKNDOvVDIBVOw+weEtypZ33iyXbGfnKXFIzsmkSH8N7o46nV8s6lXZ+CQ1KSCLVSJcmtTmqYU0APphXOZ0b3p27iZvenEdGdg5t68fx/o3H065BzUo5t4QWJSSRasTMuLCnV0v6aOFWMrJyKvR8E2at584PFpHjoHuzeN4bdTxNNCadFEIJSaSaGdqjKa2StnHb5KcJS4iHsDCoXRtuugnWrCm387z64zrun7IUgL5t6vLmdX2pGxdVbseXqseC5SW5UGJmSfHx8fFJSUl+hyJSep9/TvrQC7CsTKJysv+3PjLSW95/H84884hOMe6HtTzy6TIABrRL5L9X9KZGlN4xqu4SEhJITk5Ods4lFLRdNSSR6mTNGhg2jOiMtN8mI4DMTDh4EIYNO6Ka0msz1x9ORiceVZ9xVyoZSckoIYlUJ//+t5d4ipKZCU8+WabDz1m7h4c/+RWAQR3qM3ZEL42+ICWmhCRSnbzxRskS0oQJpT70jv1p3PzWfLJzHB0b1eKFy5SMpHSUkESqkwMHyrdcQEZWDje9+Qu7D6RTKyaCl0b0UjOdlFqE3wGISCWqWRNSUkpWLp9tyYdYtm0/y7ensGJ7CkkHM2kcH0PThBqs3HmAeRv2AfDUJcfQsl5ceUcu1YASkkh1cvnlMG5c0c12kZEwYgRZ2Tn8sjGJb5btYOqyHazZlVrs4f90cjtNrCdlpm7fZaBu3xKy1qyBo4/2etMVIqdGLM8++T4TdkWw+8BvJ/SLDDfa1q9Jh0a1SKwZzfbkNLYkHWJ7chrHt63HmIu6E65Ru6UQxXX7Vg1JpDpp29Z7z2jYMK+WlKemlBEWTlZYBDedfSfT1+UAXjLq2KgWp3ZuyMkdG9C1aTyR4Xr0LBVDCUmkujnzTFi0yOvaPWECHDiAq1mTKZ0G8Wz3c9lYpzHtG9Tk/J5NOffoJjSvG+t3xFJNqMmuDNRkJ1XR6p0pfP3rTk5on0iXJrUxU9OblC812YlIibRrUIt2DWr5HYZUY2oMFhGRoOBbQjKzwWY23sxWmNlBM9tsZpPMrFsJ9l1vZq6QZXm+soWVG1VxVyciIqXlZ5PdKKAe8CSwDGgI3AnMNbNBzrnZRex7PhCdb103YCwwuYDy7wBP5Vu3tixBi4hIxfAzId3snNuZd4WZfQWsA/4CXFjYjs65+fnXmdnwwB9fKWCX7cUkOBER8ZlvTXb5k1FgXRKwCmhWmmOZWRQwHJjhnFtZPhGKiEhlCqpODWZWH+gKLCnlrkPxmv8Kqh0BXGFmh8wszczmmNnFxcSRVNQCxJcyPhERKUbQdPs276WHsXhJ8olS7n41cAB4t4BtbwKfAZuAxsBNwDtm1tg593RZ401OTiYhocCu9CIiUoDk5GSA2oVtD5oXY83sCeB24Crn3PhS7NcM2AC86py7tgTlw4DpQA+ggXPuUBlizcJLnPtLu28Vl1tzTPY1iupB97py6D6Xr9pAjnOuwMpQUNSQzOxRvGQ0ujTJKGAkXnIorLnuN5xzOWb2BnACXvPg3FKej8JuZnUXaM6ksLewpfzoXlcO3efK5fszJDN7GPgrcKdz7plS7mt4CWm5c25mKXbNve6c0pxPREQqjq8JycweBO4H7nfOjSnDIQYCbSlh7ShwzjDgMiAFWFqGc4qISAXwrenJzG4HHgI+AaaaWd88m9Nz3zUys+nAQOdcQSM9Xg1kAa8Xco47gA7ANGAb0Ai4ERiA9x5UWrlcjIiIHDE/n4WcG/g8J7DktQFoVdTOZlYL7+XZz5xzOwoptgI4D69beAKQCswDhjjnPi5b2CIiUhGCppedhD49AK48uteVQ/e5cvneqUFERASUkEREJEioyU5ERIKCakgiIhIUlJBERCQoKCGJiEhQUEKqpsysmZk9bWYzzOxAYFr3QRV0ruFmtjAw/cdmM/unmcXkKzOyiOnmYwo7digIwns9xMy+NbPtZpZuZtvM7ON8L6eHnCC8z13M7AUz+ylQzplZq4qIp6pQQqq+2gGX4k3b8U1FncTMLsebAuRH4EzgH8DNwPhCdrkC6JdvSa+o+CpJsN3rRLxBhW8BTgP+DNQHfjCzARUVXyUItvvcG28AgO2BslIc55yWargAYXn+PBRwwKByPkc43pBNU/Ktvy5wvj551o0MrDvG73tT1e91IfvXxkv8r/h9v6rKfc4Xz62B7a38vk/BvKiGVE0550o00rmZxZjZw2a2Kk/zznOBoZuK0xdv/MDX8q1/E8jEG/qpyguRe30ALyFlliTWYBRs97mk8cj/aF4fKZSZhQOfAr2Ax4CfgY7A34FuZnZSMf/pugY+fzMlvXPuoJmtybM9ry8CU9knAV8D9znnVh/ZlQQ/P+514JxhQFPgHsCA54/wUoKaT/+mpYSUkKQolwAnA2c75z4LrPvGzDYDk/Hazz8tYv96gc+9BWzbm2c7eO3sjwKz8aYG6Yn3JTnHzI51zq0t81WEhsq817nm4H0xg9cMdYZzbmFpAw8xftxnKSE12UlRzgL2AF+ZWUTugldzycabjwozC8+7PTDnVF6FDQdyeL1z7gvn3H3OuU+cc985557E++KoiTeBY1VXafc6jxFAH7xmpiXA5xXVKy2I+HGfpYSUkKQoDfF+48vMt6TiPdxNDJT7Jt/23AkT9wQ+C/qtsS4F/5Z5mHNuCd50If3KfAWho9LvtXNumXPuJ+fcJLyawVrg6fK4mCDm679pKZqa7KQou4Ed/H6+qrzbAW4AahWwPndG3q7AytyNZhaLN9NvSeakCqN6TDXv6712zmWb2TxgeOnCDjnB8G9aCqGEJEX5HK/NPcc590thhZxzKwrZNBvv2dAIYFKe9ZcCkfnW/Y6ZdQV6UMiMwFWM3/c6GjgeqOodSHy9z1I0JaRqzMyGBf54bOBzoJklAqnOuc+Bt/BeVP3CzJ7Eaz5zQHPgdOBp59zMwo7vnMsys7uB8Wb2HPA+0Al4HHjfOTc7Tyxf4001vxSvC3IP4C4gGa+zQ0gLwnv9Hd5zoySgJTAKOAq4oJwu2RdBdp9j8Z5ZAXQPfJ5pZruAXc657478iqsYv1+E0uLfgvcfsaBlfZ4yUcDdwGIgDS9BLAaeAhqX8DyXB/ZJB7YA/wJq5CvzJF4y2o/XZr8RGAe08Ps+VcF7/TfgF2Bf4F5vBz4EBvh9n6rYfW5VRDzT/b5XwbhoPiQREQkK6mUnIiJBQQlJRESCghKSiIgEBSUkEREJCkpIIiISFJSQREQkKCghiYQIM2sVmAb7Ib9jEakISkhSpZnZoMCX+B1+x1JVmNlDgXuau+SY2V4z+8bMhpTT8YeWR6wSWjR0kEjo2ADUALL8DiTgAWAd3vdIW7wBSaeY2eXOuTeP4LgP4s3IOvnIQ5RQooQk4gMzq+WcSynNPs4bViWtgkIqi8+dcz/n/mBm7wML8IblOZKEJNWUmuxEAsws2sz+amZLzSzNzJLM7GMz65GvXJiZ3Wtm35vZdjPLMLONZvaCmdXLV/bwcx8zu8TM5pnZIeDZwPbxge3xgf13Bs79o5n1KexYhRz/HDObG9h/m5mNCUw+l/86LzSzhYFyG83sQTM7JXCckWW9f86bbXY30L6Ac95kZl+Z2ZbA/dpmZm+YWav81xL48cq8zYL5jnVK4FhJgWtYZGajyhq3BA/VkEQAM4sEvsCbgmEC8BwQD1wH/GhmJ+apDUQBfwE+AKbgTe52LHANMMDMejnnMvKdYijwJ+AF4EW8QWTz+hLYBTyMN/nbbcBnZtaqhDWps4CbAsd+BTgPuANvANV/5LnOS4C3gTV4g6xmAVcC55bgHEUyszpAHWBnAZvvwJu64Rm8Sey6AtcCJ5tZN+fcHrzrH4F3/38AxhZwjusD1zgbbxT4VOBU4AUza+uc+8uRXof4yO/RXbVoqcgFGIQ3uvIdxZT7c6Dc6fnW18YbeXx6nnVGvpGdA+uvCRzj4jzrWgXWZQKdCthnfGD78/nWXxRYf0MBx3qogHWpQKt8MS4BtuVZF4E3MvUOoE6e9TXxZot1wMgS3NOHAmUH482w2gjoD3wbWP+vAvaJK2Dd4ED5O/Otd8D4Aso3xmuyfKuAbU/jTUHe1u9/c1rKvqjJTsRzObAcmGdmibkLXm3oa7yaTw3wnuU45w4BmFm4mSUEyk4LHKtPAcf/1Dm3rIjzP5nv59xj/a75qxCTnXPrc39w3rf0t0AjM6sZWN0LaIL3Zb8vT9kDeLWO0pqKV6vZBszAm2r+ceCv+Qs651LhcHNnfOB+LcSb+qGg+1WQYUA08HLev6PAsT7GewQxuAzXIUFCTXYink54Pdh2FVEmEdgEYGYXA7fjTSQYma9cnQL2XVnAurzW5v3BObfHzMBrviuJtQWs2xP4rIc36WHrwM8FzYZa2AypRbkZ77pigZPwmiTrOOd+1wvQzE7G65XXB4jJt7mg+1WQToHPqUWUaVjCY0kQUkIS8RjehGu3FVFmF4CZXQC8A/wEjMZLUmlAON5zqIJaHg4WdXLnXHYRcZVEYfvnPUZJj1VSP7n/PVf7yMx2AI+Z2Xzn3OEal5kdC3yFNz363XhdxQ/hNc1NpOSdq3LjvwKvVlaQghKzhAglJBHPKqA+MM05l1NM2RF4Cegk59zhRGNmHSswvvKwLvDZoYBtBa0rrX/jPUd7xMzecs7ldtwYjpesz3TO5caAmcVR8toReH9HALudc0XVkiRE6RmSiOd1vIfzBdaQzCxvU1A23m/3YXm2G3BfRQZYDn7Gq1mMDPSIAyDwjOmIu0075zLxevTVw2u+y5Vbe8tfQ/srBX8HHQDqFrD+Xbwpw/+W+zwvr8CzqejSxi3BQzUkqS4Gm1n+Zxfg/bb9Il4vrVOBMYHnHdPwuma3wHtQnob3nATgfeBCYJqZvY73DGko3rOUoOWcywoMofQm8JOZvYzX7Xsk3vOm1niJ9khMwHtWdJuZPeucSwY+xOvF+JmZjQUy8O710XjvLeU3GzjFzO7C6+HonHMTnXObzexGYBywzMwm4I1eUR/ohvd30BlYf4TXID5RQpLq4ozAkt8K4EXnXKaZnY33Ls8IvHd0ALbiPSt6LXcH59xEM6uF9yX7BN67Ph/jPR/ZQxBzzr1lZll4tbm/4XUBfxlYBEzCe7ZzJMfPMrN/4vXauxX4m3PuRzO7ELgf+HvgHFOBgcD3BRzmJuA/wL1ArcC6iYHjv2pmK/Hea7oBSMBLaisCx99+JPGLv8zrHSoi1ZmZ3Y6XXPs552b7HY9UT0pIItWImUUB2Xl79QWeIS3Cewm4ifv9KBMilUJNdiLVSxvgczObiNfrrjHe0EGtgRuVjMRPSkgi1csuvE4DlwEN8Do1LAbuds6962dgImqyExGRoKD3kEREJCgoIYmISFBQQhIRkaCghCQiIkFBCUlERIKCEpKIiASF/wfap9wv/Zx4nQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_LSL.lr_find()\n",
    "learn_LSL.recorder.plot(suggestion=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-02T07:38:36.233428Z",
     "start_time": "2020-07-02T07:21:32.761863Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>seq2seq_acc</th>\n",
       "      <th>bleu</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2.141443</td>\n",
       "      <td>2.071022</td>\n",
       "      <td>0.567044</td>\n",
       "      <td>0.400781</td>\n",
       "      <td>01:35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.898499</td>\n",
       "      <td>1.855029</td>\n",
       "      <td>0.565619</td>\n",
       "      <td>0.399401</td>\n",
       "      <td>01:35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.753499</td>\n",
       "      <td>1.721894</td>\n",
       "      <td>0.570702</td>\n",
       "      <td>0.409064</td>\n",
       "      <td>01:38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1.610226</td>\n",
       "      <td>1.642993</td>\n",
       "      <td>0.572891</td>\n",
       "      <td>0.405655</td>\n",
       "      <td>01:39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.497186</td>\n",
       "      <td>1.608099</td>\n",
       "      <td>0.599936</td>\n",
       "      <td>0.430033</td>\n",
       "      <td>01:36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>1.357414</td>\n",
       "      <td>1.595815</td>\n",
       "      <td>0.614662</td>\n",
       "      <td>0.440933</td>\n",
       "      <td>01:36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>1.260690</td>\n",
       "      <td>1.604014</td>\n",
       "      <td>0.640098</td>\n",
       "      <td>0.473974</td>\n",
       "      <td>01:35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>1.187641</td>\n",
       "      <td>1.621607</td>\n",
       "      <td>0.664208</td>\n",
       "      <td>0.498927</td>\n",
       "      <td>01:36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>1.166813</td>\n",
       "      <td>1.645855</td>\n",
       "      <td>0.676659</td>\n",
       "      <td>0.511069</td>\n",
       "      <td>01:35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>1.142947</td>\n",
       "      <td>1.658493</td>\n",
       "      <td>0.679379</td>\n",
       "      <td>0.513981</td>\n",
       "      <td>01:35</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/skumar/miniconda3/envs/ImageCaption/lib/python3.7/site-packages/ipykernel_launcher.py:34: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEDCAYAAADX1GjKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3wUZf7A8c83lVDSCIQqoXcQUESqVAURFb1TsaCC/mxn1/NUFNvp6Z29KwienooKpyjqiYggRUCQ3nsJLZCQkL77/P6YTdjdbDa7IZvNJt/36zWv3X3mmWeemWzmuzPzPM+IMQallFI1W1iwK6CUUir4NBgopZTSYKCUUkqDgVJKKTQYKKWUAiKCXYHyEJFCrEB2Ith1UUqpEBIL2I0xJY79EopNS0XEDkhsXBwS7MoopVSIyMjIADDGmBJXhULyzAA4IdF14tbvPECzhNrBrotSSoWE+Ph4MjIyPF5RCel7BvmF9mBXQSmlqoXQDgY2DQZKKVURQjsY6JmBUkpVCA0GSimlNBgopZQK3dZEAOTpPQOlqgRjDEePHiU3Nxe7Xf8vK1NYWBgRERHExsZSp06dcpcT0sEgO88W7CooVeMZY9i/fz+ZmZlER0cTHh4e7CrVKAUFBWRnZ5Oenk69evVo0qQJYWH+X/QJ6WBwPDs/2FVQqsY7evQomZmZJCcnk5iYGOzq1Eh2u520tDSOHj1KRkYGCQkJfpcR0vcMjp/UYKBUsOXm5hIdHa2BIIjCwsJISkoiKiqKrKys8pVRwXWqVMf0zECpoLPb7XppqAoQESIiIsp9zyakg8HBjNxgV0EppaqFkA4Gu9Oyg10FpZSqFkI6GOw5lk0ojrqqlFJVTUgHg6y8QtL0JrJSKkCWLl3K5MmTSU9Pr/CyRYTJkydXeLnlFdLBAPRSkVIqcJYuXcoTTzwRkGCwZMkSJk6cWOHlllfIBoMwx1NtdqedDG5FlFI1nt1up6CgwK9l+vTpQ7NmzQJUI/+FcDCwosGafRlBrolSqjqaPHky99xzDwAtW7ZERBARdu3ahYhw991388orr9CmTRuioqJYtGhR8XJnn302CQkJxMXF0bt3b2bMmFGifPfLRJMnT0ZE2LhxI1dccQWxsbEkJydz4403Fj2hLKBCtgdyuOPUYMHWI0GuiVLKXaHNTmoVavrdOK4WEeH+/fadOHEiGRkZvPzyy8ycOZPGjRtbZTleP//8c5o2bcpzzz1H3bp1ad26NQB79uzh9ttvp3nz5hQWFvLzzz8zbtw4MjMzmTBhQpnrHTt2LFdccQU33XQTa9as4eGHHwZg6tSpftXfXyEbDCLCrWCw48hJ9h3P1sdfKlWFpGbkMuD5n4NdjWILHxxM80T/jhHNmjWjRYsWAPTo0YOUlBSX+bm5ucydO5fY2FiXdOeDtt1uZ+jQoRw9epTXX3/dp2Bw8803F5+RDBs2jO3btzN16lSmTJmCSOCe+h6yl4nCRYivHQnA/M16dqCUqlxDhw4tEQgAfv75Z84//3ySk5OJiIggMjKSKVOmsGnTJp/KHTNmjMvnbt26kZuby6FDhyqk3qUJ2TMDgH6tk/h2bSqLtx/lmj4tgl0dpZRD47haLHxwcLCrUaxxXK2KL9NxucjZ0qVLGT58OEOGDOGNN96gadOmREZG8tZbb/l8mad+/foun6OjowHrTCSQQjoYDGrXgG/XpvLzpiNk5xdSOyqkN0epaiMiPMzvyzKhxtMlm88++4zIyEhmz55dfBAHyM+v+v2hQvYyEcD5nRsRGS7kFNiYt+lwsKujlKpmig7oOTk5PuUvGizO+XkChw8f5quvvgpI/SpSSAeDuNqRDGjbAIBZK/cHuTZKqeqmS5cuALz++ussWbKEFStWeP2Vf+GFF5KVlcXVV1/N3Llz+eijjxgwYADJycmVVeVyC+lgADC6m3Xd7qdNhzmSmRfk2iilqpMBAwbw0EMPMXPmTPr378/ZZ5/NgQMHSs0/dOhQ3n33XVatWsXo0aN56qmnuPvuu7n66qsrsdblI6E40JuIpMfFxcWlp6dzMq+Qzo//AMDjF3Xihn4tg1w7pWqW3bt3AxQ3w1TBU9bfIj4+noyMjAxjTLz7vDLPDERkqIhME5HNIpItIvtEZKaIdPVh2ckiYjxMB8veLN/UiY7gqt7NAfhy5b6KKlYppWoUXy4T3QKcAbwEjATudXxeLiJ9fFzPcOBcp2mU/1Ut3WU9rfE91u0/waaDJyqyaKWUqhF8aYt5uzHGpamOiPwP2Ak8AFzmQxkrjDEVP+yfQ68WCaTUr82utGw++W0PT1zcJVCrUkqpaqnMMwP3QOBISwe2AlViyD0R4dIeVlWmL9nNiVz/Rg9USqmarlytiUSkAdAFWOfjIhtFxCYiqSLynog0LKP8dG8TEOe+zJ/OOhWXJk5b4cfWKKWU8jsYiNXt7l3Hsv8sI/t24GHgBqz7Bm8CVwJLRSTB33V70yQ+hjsGtwFg2a5jrNWhrZVSymflOTN4AbgEuMUYs9FbRmPMv40xzxpjvjfGzDPGPAWMBVoCt3tZLt7bBHg80t81rG3x+ye/WU+hzV6OzVNKqZrHr2AgIs8A9wF3GWOmlWeFxpgfgVSsVkUVKjI8jH9P6A3A8l3HufiNRRW9CqWUqpZ8DgYi8iTWJZ8HjTGvVsB6A/KzfUDbBtw8sBUA6w+c4PfdxwOxGqWUqlZ8CgYi8jgwCZhkjHnhdFYoIiOAZGDp6ZTjzb3D2xW/f+nHLYFajVJKVRu+9EC+D5gMfAPMFZE+TlMPp3zzRcS4LbtKRO4RkVEiMtwRVL4EtgFvVOiWOKkVGc4HN5wNwK/bjrJ429FArUoppXxy/fXXuzwtrehZytOmTfN72UDwpdPZRY7X0Y7J2W4gxcuym4DbgCZAJLAXeB94KpCd0ADOa9eAHmfEs2pPOvfOWM1P9w2iTrQ+70ApVTU0btyYJUuWFD87Odh86XR2njFGSplS3PO5LXuVMaatMaaOMSbKGNPaGHOPMeZYALbFhYjw90u7EhkuHDyRy9RfdwZ6lUop5bPo6Gj69OlDgwYNgl0VoBoMYe1Nx8axXNsnBYB//biF1XsDejKilKpGZs2ahYjwyy+/lJj39NNPExERwYEDB/jxxx8ZM2YMTZs2JSYmhnbt2nHvvfdy4oT3cdJKu0w0depU2rdvT3R0NO3bt/f5cZmnq9pfN7llUCs+XLKLQrvh4jcWcdt5rXnwgg7BrpZS1ZutEE5UoQdOxTaFcP8Od6NHjyYpKYlp06YxaNAgl3kffvghw4cPp0mTJnz99df079+fm2++mXr16rF161aeffZZli9fzsKFC/1a55QpU5g4cSKXXnopL774IsePH+fxxx+noKDA5elpgVDtg0HD2Fo8NLIDT39r9Y97c/52msTHcE0fHXtdqYA5sR9e6RbsWpxy1xpI8O9/PjIyknHjxjF16lRef/116tSpA8CiRYvYunUrTz/9NAC33HJL8TLGGPr160f79u0ZOHAgq1evpnv37j6tz263M2nSJHr37s2XX35Z/Izlvn370r59e5o2bepX/f1VrS8TFZk4oBVz7z0V2R/97zqW7wr4bQulVIi74YYbyMrK4osvvihOmz59OgkJCVx88cUAHDx4kDvvvJOWLVtSq1YtIiMjGThwIACbNm3yeV2bN28mNTWVcePGFQcCgFatWtG3b98K2qLSVfszgyJtGtblg+vP5oZpywGY9N91zLqtHzFR4UGumVLVUGxT69d4VRFbvl/VZ555Jt27d2fatGmMHz+enJwcZsyYwdVXX010dDR2u53hw4dz+PBhHnvsMbp06UKdOnXYu3cvY8eOJScnx+d1paWlAdCoUaMS8xo3blz8FLNAqTHBAGBwh4a8dXVPbv14JZsOZtLxse/58Z6BtE2uF+yqKVW9hEf4fVmmqho/fjz33Xcfu3btYsmSJWRkZHD99dcDsHbtWtatW8f06dO57rrripfJyPB/oMz69esD1pmGu9TU1PJV3g814jKRs5FdG/PgBe2LPw9/aYE+HU0pVaprrrmGiIgIpk+fzvTp0+ncuTNnn211ai26nBMVFeWyzHvvvef3etq3b0/jxo35z3/+45K+Y8cOFi9eXM7a+65GnRkUuXVQa1bsOs68TdZzey54eSFxMZH847KuXNClcZBrp5SqSho0aMDIkSN55513OHz4MM8991zxvI4dO9KqVSseeughAOLi4vjkk0/4/fff/V5PWFgYTz31FBMnTmTs2LFMmDCB9PR0HnvsMRo3DvxxqcadGYAVzadefzZje5y6jpiRU8AtH61kinZOU0q5ueGGG0hNTUVEuOaaa4rTIyMj+frrr2ndujU33XQT1157LeHh4Xz66aflWs+ECRN4//33WbduHWPHjuXJJ5/k0UcfZciQIRW1KaUSY0zZuaoYEUmPi4uLS08//U5kr8zdyktzXQeze+2qHlzUvclpl61UTVB0Y7NFi+pxjyCUlfW3iI+PJyMjI8PxXBgXNfLMwNldw9qy7OGhfDzxnOK0v3yyit1pJ4NYK6WUqlw1PhiA1TGtX5skvr97QHHaoBfmsz/d92ZhSikVyjQYOOnQKJYXLj/Va7Lfc/NIeehbQvFSmlJK+UODgZvLezWje3PXy2nTF+8KTmWUUqqSaDBwIyL897a+vPjnU+OJTJ69gZV79PGZSqnqS4OBByLC2J7N+PWvg4vTxr65mCveWUKBLSCPblYqZIWFhWGz2YJdDYU12J3zuEb+0GDgRbOE2vzfwFbFn3/beYy2j3zHlkOZQayVUlVLREQEBQUF2O36QymYbDYbubm51KpVq1zL1/h+Br7YciiTES8t8Dhv1aThJNSJ8jhPqZrg5MmT7Nmzh6SkJJKSksr9y1SVn81mIzU1lczMTFJSUoiJifGYz1s/Aw0GPrLbDa0enlMi/fzOybxz7VmVVg+lqqJ9+/aRmZlJVFQUERE1cpSboLHb7eTl5WGMITk5mcTExFLzaqezChAWJux67kLevbaXS/oP6w+RkVMQpFopVTU0adKERo0alRiwTQVeeHg4iYmJpKSkeA0EZdEzg3JKz87nrKfnUmg3DG7fgA9u6B20uiillC/0zCAA4mtH0b9tEgA/bz7CZ8v3BLlGSilVfhoMyuvgOqZ220inhtEA/PXLtVzyxiJyC7SJnVIq9JQZDERkqIhME5HNIpItIvtEZKaIdPVlBSLSWkT+KyIZIpIpInNEpNPpVz3IFrxA2Ow7+a/tDm4M/44YcvljbzodJn1Pdn5hsGunlFJ+8eXM4BbgDOAlYCRwr+PzchHp421BEWkILARSgPHAVUAi8IuINCt/tYPMboeoOiDhRJ1M5bHIf7Mo+k7+Ej6TWLKY9N/1wa6hUkr5pcwbyCLS0Bhz2C0tHtgJzDPGXOZl2eeBvwCtjTEHHGn1Hct+bIy5tVyVrgI3kAE4vhsWvwYrPwRbHgBZphYf2YZzuNONPDYu8A+kUEopXwWkn4GILAOMMeYcL3m2AhuNMWPc0j8Ghhljksu57qoRDIpkHoKlb2KWT0Hyrd7JeSaSGbZBJJ3/ACMHeD2BUkqpSlHhrYlEpAHQBVjnJU8M0LqUPGuAho7LSKGvXjIMfwK5Zx0ZfR8izdQjWgq4NmIuw+eOZOakC8nYvSbYtVRKqVL5HQzE6mv+rmPZf3rJmgAIcMzDvKK0+qWsI93bBMT5W+9KERNP3Ii/UXjnGiYXXMd+U58IsTM2/FfiPhjAxpdGY9uzPNi1VEqpEspzZvACcAlwizFmow/5vV2HCr0ebz5Irp/I5Gdew3b7Sh4ouJnt9sYAdMxYSPjUYRx6bQTsmA8h2OFPKVU9+RUMROQZ4D7gLmPMtDKyH8c62Hv69V/UZ9rTWQPGmHhvE5DhT72D5YyG8bzwzAusvvgHbs2/i3X2FACS036DDy9m/wvnsnn+J5zIySO/UEd8VEoFj883kEXkSWAS8KAx5gUfl9kGbPBwA/kjYIQxplz3DKrcDWQfnMwr5N7PVpGz6Uduj/iKc8I2Fc/bYm/KW4VjaDd0PLcO7RjEWiqlqrPTbk0kIo8Dk4FJxpinfV2xiLwA3AG0NMYcdKQlYjUt/cQYc4uvZbmVG3LBoMgHi3byxOwN9JLN3BbxNUPDVxXP22eSWNBgHFfe9DfComsHsZZKqerotIKBiNyHdaP4G+AZt9l5xphVjnzzgUHGGHFaNhlYDRwAngAKgUeBdkAPY0y5BvQJ5WAAkJlbwJZDWRTa7Dw//QvG22dxYdhSwsX6WxwxcextfwM9x94HtWIByMorpG60Dg2slCq/0w0G84FBpczebYxJcc7nHAwc6W2xgslgrHsUC4H7jTHl7qYb6sHAk0M71zNvyiNcFr6AKLHGNzphajPdNoIPCi/gGFZQWDlpOAm1IxERFm87StrJfC7q3iSYVVdKhQh9uE2IyMguYMa8pcjSNxgXPo/aYvVqzjFRfGobzLuFo0n1cD/++r4pTB7TubKrq5QKMRoMQsyXv+/j6c8Xcn3E/xgf/gPxchKAfBPOLNsA3rZdxE7T2GWZe4a1465hbYNRXaVUiNBgEKIKbHaWbdpNt0OzqLvybSTrEAB2I8yx9+atwotZb12lo250BGsnj9DnzyqlSqXBoDooyIXV/4FfX4b03cXJ+5P68ciBfiyxd+Kt6/sypEO5hntSStUAGgyqE1shrJ8Fv74IhzcUJ2ebaBbZu3DWiCtJ6D4aYvWmslLKlQaD6shuhy3fw+LXMHuWIO4jeyR3hXYjoO350OwsCAsPTj2VUlWGBoPqLusI7097l0aHFjAwbA2xku06PyYR2gyDdudD6yFQO9FzOUqpak2DQQ2QlpVHr6fnEkEhNzQ/zCNt98DW/8GRTa4ZJQyanwNtR1jBoWEn0JvOStUIGgxqiKvfX8qibWkAPDGmM1f2bk505j4rKGz5AXYthMJc14Vim526nNRyIETpMBhKVVcaDGqIdfszGP3ary5p949ox63ntSE8TCA/2woIW36wAkTGXtcCImpBygDrjKHtCEhoUeq6vlubyq0frwRg45MXEBOl9ySUquo0GNQgG1NPMPKVhR7nXd83hUXbjvLEmM70bV2fE3vWcHLdHJIP/oLsW4YYm0t+e1J7diX2p/k5l5LbqBf16tTGGEPHx74nt8B1yO03xvXkwm6uHeGUUlWLBoMaZuWe44x9c7Ffy8SSxaCwNQwO/4Pzwv4gUbJc5p8wtVkfcxa5LYdx36qGxWMlOVv44GCaJ+plJqWqKg0GNZAxhu/WHeQ2x6Ucf4Rh50zZxuDwPxgStorOYbtd5tuNsNq05njTwfxrVwobTAsMYUzo35JJoztV1CYopSqYBoMaLrfAxucr9jLpq/UM6dCQeZsO+7V8Msc4L3w1Q8JW0T9sLXUcA+gVSZNE5hV0Ia9BV665eJTVQkmbrypV5WgwUC7yC+0U2u3UjrKej2CzG8IE8grtLNmRRmytSI5k5nJBl8ZsPZTJsZP5ZOUVMmH6CqIooHfYJj7oe4zIHT/CsR2eV1I32QoKDTtBcido2BEadICoOpW4pUopZxoMVOAc3cbRVV+zcsE3tJe9tAjzdtYhkJDiGiAadob6rSE8srJqrFSNpcFABVzKQ98C0D5B+OGaZDi0AQ5vhMPrrVfHiKsehUVCUjunAOE4o4hrDmFhlbQFSlV/3oKBPkdRVYjPbzmXP729hM3HDeulDZ179nLNcDLNGljPOUAc3gh5J8Be4Ehze/hdVF1HcHAKEA07Qd0GlbdhStUQGgxUhTirRQJtGtZl2+Esvvh9H52bxLlmqFMfWg6wpiLGQMY+1wBxaAMc3Qy2fMjPgn3LrcmlrAYlA0TDDhBdL/AbqlR52e1gy4PCPOv7Xfya6yEtz5E3/9RrYW7JNJfXPM/LOa8z70Sp1dPLRKrCvPrTVl78cQtN4mqx8K9DrF7P5VBQkM+6tSvpFnmA8KObMIfWI4c3Om5We/m+xp8B8S2slkwxiVC7vmNKPPValB5dT8dkUmWz260fJflZkJdZcsrPsg6weZmQl+U9vTAn2FtD/HMnyMhD7xmowNp59CSD/zkfgOcv78afz2ru03JF38HcAjvrDmTwp7eXlMjTrVkcX9/c0zprOLSBozv/IHXLSjqF7yP85EH/KxsW6RQk6kNMgofgUd8RPDSAhAy7DQqyoSDHacp2Opg7HaS9HuCd3le28GiIiIbwKGuImIgoR1qU2zyn14jo0ucVv0YTP+BGMjKz9Z6BCqyWSaeajT74xRrSs/M5v3MjWtSvg81uOJlfSGytkq2GHp61lk+W7S2R7mzNvgw+/P0wO45EcVbKYO5YFg+cR+2ocF67JIV3Pv+GdmH7+L8eMTSvlQvZaZB9zDGlWZO94FSB9gLrpra3G9vunAOIc5BwDx4xCU7/hKX8A9e0oGK3W7+Miw7OHl/dDuAl5jmlFeZ6zmfLD/y2SDhE14XoWOsHQlRd6zW6npf0elaz6qLvgttBuvh7Eh4Z2O9G2P+Vvll6ZqAq0h3/Wck3a1Jd0i7q3oS0rDwWb0/jtvNa8+AFHQDrjKDl3+ZUeB12PXdhyURjrF957kEixylYFAeOY6fSA3VwCXf/tRdVSlppvwgjvfwSdCxjDNgLrclW4OF9gfXkvOL3BdYva7tjvq3Q6b2n5X0o15bvOEjnlb1PAi3K+YBdz+mAHesh3T1v7KmDe2RMyAZzbVqqKlVRM1Nv6kSFc9PAVrw8d6vH+Y+N7kTXZnFMmLacs1MS+cmPXtNf39GPbs1KfNf9ZwzknzwVLHKOuQYMj+kBDCDVVVgkRNa2DrKRMV7elzbPLS2iltu8WhBZR5spo8FABcHtH6/k27WpZWd00vOMeF4b15Om8TEu6Xa7odXDJc8gZt/Rn4te/7VEOpRydlAZjPG9dUhxK4+CUlqZlLKsrcBLeU7lilgH2vBI67GnYZEQFuH4HOHj+0gIjzj1PizcMa+orAi3csOd1ulI83QAj3A6wGuHw0pz2sFARJoBDwC9gDOBOsBgY8x8H5adBoz3MOs3Y0yfMlfuuUwNBiHivQU7eGbORq95bhrQkkcuLP8Adza7QcAlYOgIqkqV5C0Y+Hre1Aa4CsgCfipHHbKAc92mCeUoR4WYmwa2Yv795zHunDPY+sxINj11AfXrRLnkeXhUx9NaR3iYEBYmrH5sRHHagOd/Pq0ylappfG1NtMAY0xBARC4Bxvi5HpsxZqmfy6hqIiWpDn+/tCsAkeHw+6ThgPWLvrx9ETyJq+16uWHb4UzaNNSOaEr5wqczA2OMvexcSvmnIgNBkTl3nurh/PS33i9PKaVOqazb63VF5JCI2ERkt4j8S0TqVtK6VQ3SqUksA9omATB/8xHe/mV7pa37cGYu/Z6bx30zVlfaOpWqKJURDFYD9wNXA6OAz4E7gHki4rEZgYike5uAOE/LKQXw0MgOxe+f+24TOfk2L7krzvkvLWB/eg5frtzHom1HK2WdSlWUgAcDY8xLxpiXjTFzjTE/GGPuxwoGZwNXBHr9qubp3CSOy3s1K/7c8bHvyS0ITEAoao23ePtRjmef6uH83Tr/mtUqFWzBGo7iI+BtrFZFH7nP9NTsyZmeHaiy/PNP3fni933FnztM+t5j3wNjDPuO5xS3Ppp5W1/ObBZPmIf7GfmFdj5bvocuTePo3izeY9+HIsdOasczFVqCFQyK/tP0xrQKmFev6sGdn6wq/vzxb7upHRXOPZ+tZvqNvWmWEMPQf/3isszYNxfzwPntuX1wmxLl3fnJKr5f79ugeJtSM0+v8kpVsmAFg2uwLlFpc1MVMGO6N6FZQgxj31wMwCOz1hXPGz91WanLvfDDZv50VjMa1qvF2n0ZxMVE8sAXq/lt57Ey1/nRhHO4Zspv7Eo7SU6+jZio8NPfEKUqgc/BQEQud7w92/E6SESSgJPGmO8ceXYBGGNSHJ9bAP8GPgG2A+HAMKx7BkuAz057C5Tyokdz38coOiOxNnuOZQPQ+5mf6N4sjtX7MrwuM/3G3jz1zQb6t0li8pjOHM2yBmSzG3hv4Q7uHNq2/JX3whjDku1pdG0WRz0PI8Eq5S+fxyYSkdIy7nY6+O8Cl2CQAEwBegLJWJeHdmC1KHrOGFOupz3ocBTKHzNX7uNeL809tz4zkshwqy2FL4PsPX9ZN848I562DesiHkavbPvIHAps1r/L9r+PqvD+FOe/tIDNh05dhvrt4aEkx9aq0HWo6kkHqlPKSV6hjW9WpzJ7zQHuHtaOM53OHn7efJgbPljucbkX/9ydsT2beZznbOL0FczdeOo5CTufHeUxaHhTaLOzPz2HFvVPPSNi2+Eshr34S4m8Z6ck8Pktff0qX9VMGgyU8sMHi3byxOwNADSJq8Ws2/uxIfUE57Vr4PNB3f0MY+OTF5R6/8BuN9iNIUyEL1buY/bqAyzcavVTePTCjkzo35KPlu5m0lfrS13fooeG0CSult9BR9UsGgyUqmSr96Zz8RuLij83jqvF7L/054f1B7nkzKbUibZu1xXa7LR55Du/y795YCvyCmxMX7LbJf3uYW05OyWRuJhI2iXXIypCx/BXp2gwUCoIXpm7lZfmbvE4b2yPpvzrz92ZsWIvf/1yrV/lrpo0nATHyK9l3eP45i/96dJUu+QoiwYDpYLkbzPX8smyPR7nNUuIYd9x/9pQTL+xN4PaNSj+nJqRw7nPzvO6TNAe9KOqHA0GSgWRLy2UepwRz6zb+rmkffH7Pu7/3GoFtXbyiFKbkOYW2MjOt9HzqR89zl//xPnFl6VUzabBQKkgyi2wcdlbi4mLieS6c1twy0crS+QpbaiM7UeyaJ5Ym+iIsjuvrdufwR970xlzZhMOn8hzaXnk3HxW1VwaDJSqQvYey3Z5Elug+gm4n5H8/ugw6teNrvD1qNBREY+9VEpVkOaJtblzSBuu7dOCTU9dELAOY29e3dPlc6+n5wZkPap60GCgVBDcO6I9T13ShVqRgRu7aFTXxozq2sgl7S9OA/d5UmCzFw+pAfDZ8j08O2cjdnvoXUFQ/tHLREpVc5+v2MsDX6xxSXtiTGf+89se7hrWllFdG5OWlcf4D5axbv+J4iMxLAYAABG/SURBVDyf3NSHq947NZZk0/gYrju3BTNW7GX7kZPcNKAlj1zYyeM6s/IK6fL4DwC8cuWZXHxm0wBsmfKX3jNQqoa77ePfmbPW8/Db02/s7XUUV29+vv88WiadGjIjI7uAq95byobUEy755t9/HilO+VRwaDBQSvHpsj08NNO/Dm6+qBUZRm5B2Y8m0f4Owac3kJVSXNn7DD65qY/XPPcMa0e75LouaSO7NKKRl5vcvgQCsFo3pTndj3BnjOHdBdvp++xPpXbUU4GjZwZK1TDPfbeJt3/Z7pLWPrkeH990Dkl1o8kvtPPNmgMMaNuABvU8N0XdmHqCka8s9LqehvWi+em+QXSd/D+X9Au7NuaFP3WjdpTVEW7G8r08+OWaEsuveHQYSRXQFPZIZh5JdaMqdBC/QpudnAJbyD1LQi8TKaVKWLbzGH9+Zwljujfh1at6+L18Tr6Nez77g8XbjzK+bwqvzdsGwGU9m/Hs2K7Fg+S9v3AHT3+7scTyvVMSWbar9KfHjeiUzLvXneWStictmy2HMhnWKdnjMsYYVuw+TsukOpzl1pS2op4t4Ty44Kzb+tLjjAS/ls8vtLPn2ElaN/D8PIxA0mCglPIot8BGdERYQA9Kxhium7qseFhuX/Vvk8RHE88p/uzcWa9pfAz703MY26MpL15xJjuOZDFnbSrJsbVKtJxyd/+IdtwxpPxPoHt41lr+85t1Gat1gzr8dN95LvONMew7nsOv245yTstEWibVcdm/T87ewNRFOwHY/PQFpfYuL7TZifCz1/iqPcd5b+EOHjy/AylJdfj4t938/duN9G+bxI39WnJ+z9YaDJRSwVXWDezx57bg0dGdePPn7cWjvTo3S/VljCdf/XD3QNo3qlf82RjjU0DMyCmg+xOnLntFRYSx+akLXJa997M/mLlqv8fl10weQTeny2Y9z4hnptuYVJm5BcWX1kZ1bcSbV/fyaZuMMbT82xyvedLfvlpvICulguvK3mew67kLeWNczxLz7h3ejicu7kJkeBjdmp8acvuuT//Abjc8Mbv0B/t407d1fSI8XBo6/+UFzFq1jy9+30fKQ9/S8m9zOJJZ+s1tgKvfX+oSCMC65PO64/IYWL/mSwsEgEsgAFi5x/pBm1tg4+vVB/ho6W6Xp+TNWXuQlIe+ZfriXeQX2jmRW8CKXceYu+EQv249St9nf2Kr4xGorR72HggATuQWljpPzwyUUkFz7GQ+u9JO0tPpursvv3B9UfSruujeSFlaN6jD5DGdGdC2QYl5v+8+xmVvlV7GtmdG8sfedK6dsoycAttp1bs0A9s1YMGWI6dVxp6Xr8DkndTLREqp0ODtCXA7nx0FQL7Nzn0zVvPNmlQA3rm2F9ERYZyVkkhdtyG7s/IKWbc/g8ZxtRj0wnyv676+bwp/vaADn/++l42pJ3hsdGc6Pva9S57Vj4/gmzUHeGTWulLL+e6uARzNyuPaKSU79LVqUIcdR056rUdF+eYv/Rn92q+ABgOlVAiaOH05czcedklb/dgI4mqfXnPOgxm59Hn2p3IvX9R5zmY3tC7l0ky/NvX5eOKpPh0FNjttnYKb+70DT/q2rs/i7Wl+12/H30dhgHcX7GB4p4a0aViv+DGsGgyUUiHH+Wbtjf1a8uiFHQmrgKahYB3IP12+hw6NYmmeGEPvZ3wLDu4PGXrxxy28+tPWEvmWPTKUhvVcO+odSM/hmW83ct+IdrRqUJeTeYXc+ckqLu/VjMEdGvLSj1sIDxPenG/1Adn6zEhyCmxsOHCCK989NUbUC5d3409nNWff8Wz+8f1mZq8+4HW9zrRpqVJK+cBbi6UZ/3cuvVsmuqS5/+L/U69mPHhBh1I76/ni2Ml84mIiXfpEzFmbym0fr+TZsV25qvcZxenOZyc3D2zFw6M6ei1bg4FSSvlgT1o287cc5rGvrNZLqyYNJ752pNdmp7/vPs6mgycY1/uMSu9EZq3/GL9sOcqtg1oTE+V9SPTTDgYi0gx4AOgFnAnUAQYbY+b7UlkR6QU8D/QB8oEfgPuMMaW3wfJengYDpZTyU0UMVNcGuArIAvy68yIiHYH5gACXAzcBPYD5IlLXy6JKKaUqSUTZWQBYYIxpCCAilwBj/FjHE0AmcJEx5qSjjHXAeuB24B9+lKWUUioAfDozMMb4NkatGxGJBEYDXxQFAkd5m4ClwGXlKVcppVTFCvRwFK2AGMBTz4w1QJcAr18ppZQPfL1MVF71Ha+exqk9BsSISIwxJsd5hoiUdWc4roz5Siml/FBZA9V5a7IUem1blVKqmgn0mUFRX+r6HuYlAjnGmFz3GZ6aPTlznDno2YFSSlWQQJ8Z7ABy8HxvoCue7yUopZSqZAENBsaYAuBb4DIRqV2ULiLtgHOBmYFcv1JKKd/4fJlIRC53vD3b8TpIRJKAk8aY7xx5dgEYY1KcFn0cWAZ8LSL/xOq9/AywC3jjNOqulFKqgvhzz+Bzt8+THa+7gZTSFjLGbBCRwVidy74ECoD/YQ1HkenH+pVSSgWIz8HAGFPmCExuZwTO6cuBIb5XSymlVGXSZyArpZTSYKCUUkqDgVJKKTQYKKWUQoOBUkopNBgopZRCg4FSSik0GCillEKDgVJKKTQYKKWUQoOBUkopNBgopZRCg4FSSik0GCillEKDgVJKKTQYKKWUQoOBUkopNBgopZRCg4FSSik0GCillEKDgVJKKTQYKKWUQoOBUkopNBgopZTCx2AgInVF5FURSRWRHBFZISJjfFhusogYD9PB06+6UkqpihLhY75ZQE/gQWAncD0wS0QuMsbM8WH54UCW0+d8fyqplFIqsMoMBiIyChgGjDXGzHKk/Qy0Av4F+BIMVhhj0k+nokoppQLHl8tElwIZwFdFCcYYA0wHOohIpwDVTSmlVCXxJRh0ATYYY+xu6Wuc5pdlo4jYHPcc3hORht4yi0i6twmI82GdSimlfOTLPYP6wBYP6cec5pdmO/AwsArrPkE/rPsOQ0WklzHmuB91VUopFSC+3kA25ZlnjPm3W9I8EVkK/A+4HXi6lOXivVVGzw6UUqpi+XKZKA3Pv/4THa/HPMwrlTHmRyAVONef5ZRSSgWOL8FgPdBRRNzzdnW8rivnet3vQSillAoSX4LBLCAeuMgt/TpgszFmgz8rFJERQDKw1J/llFJKBY4v9wzmAD8DU0SkPlans/FAf+DiokwiMh8YZIwRp7RVwIfAZqAA6AvcD2wD3qiYTVBKKXW6ygwGxhgjIpcAf3dM8cAGrE5os8tYfBNwG9AEiAT2Au8DT2knNKWUqjrE6j8WWkQkPS4uLi49XeOJUkr5Kj4+noyMjAxPLTZ11FKllFIaDJRSSmkwUEophQYDpZRSaDBQSimFBgOllFJoMFBKKYUGA6WUUmgwUEophQYDpZRSaDBQSimFBgOllFJoMFBKKYUGA6WUUmgwUEophQYDpZRSaDBQSimFBgOllFJoMFBKKYUGA6WUUmgwUEophQYDpZRSaDBQSimFBgOllFL4GAxEpK6IvCoiqSKSIyIrRGSMj8u2FpH/ikiGiGSKyBwR6XR61VZKKVWRfD0zmAVcDTwKXAhsAGaJyChvC4lIQ2AhkAKMB64CEoFfRKRZOeuslFKqgkWUlcFxwB8GjDXGzHKk/Qy0Av4FzPGy+P1AAnCWMeaAY9klwE7gEeDW06q9UkqpCuHLmcGlQAbwVVGCMcYA04EOZVzyuRT4sSgQOJZNA2YDY8tVY6WUUhWuzDMDoAuwwRhjd0tf4zzffSERiQFaA597KHMNME5EGhpjDntYNr2MOsVlZGQQHx9fZuWVUkpZMjIyAGI9zfMlGNQHtnhIP+Y035MEQJzylbZsiWDgqwzHlimfxDledZ/5TveZ/3Sf+a8y91ks4P7DHvAtGACYcs4r17LGGK8/+YvOHMrKp07RfeY/3Wf+033mv6qyz3y5Z5CG51//iY5XT7/8AY5jHezLs6xSSqlK5EswWA90FBH3vF0dr+s8LWSMyQF2YN1TcNcVOOLpfoFSSqnK50swmAXEAxe5pV8HbDbGlLh57LbscBFpVJQgIomOsmb6WVellFIB4kswmAP8DEwRkRtFZLCITAP6Aw8UZRKR+SLifg/gn1g3ReaIyMUiciHwLVAI/L0iNkAppdTpKzMYOPoUXAJ8inUA/w7ohtUJbXYZyx4CBgB7gX8DnwHpwEBjzJ7Tq7pSSqmKItaxPrRUlbvvoUT3mf90n/lP95n/qso+01FLlVJKheaZgVJKqYqlZwZKKaU0GCillNJgoJRSihALBqfzxLVQJSLNROQVEflVRLJExIjIeaXkHSciq0UkV0T2ichzIlLLQ75kEZkuIkdF5KSILBSRvqdTZlUhIkNFZJqIbBaRbEedZ4pIVw95h4vIUsd36bCIvCMiJVp0+PO987XMqkRE+orIDyKy3/F3PiIi80RkpIe8us9KISKTHf+ff3iYV/X3mzEmZCbgR6yxkiYAQ4APARswKth1C+A2n4c1suv3WM+UMMB5HvJd45j3JjAYuA3IBD51y1cLawiRXVhPnhuB1bEwB+hRnjKr0oQ1ZPo84BZgEPBnYAWQC/Rx268FjvzDsHrUpwKLgLDyfO/8KbMqTVhPL3wVuNKxDZdi9ScywJW6z3zah50d/0MHgT/Ku43B3G9B34l+7OxRji/npU5pAvwKbAx2/QK43WFO7y/BQzAAwh1fhK/c0m9y5D/HKe02R1pPp7RorHGkvitPmVVpAhp6SIvHGjjxS6e0ZcAqt/073LFtV5Tne+drmaEwYY1ovBeYp/uszH0VBiwFXgPmUzIYhMR+C/qO9GOHv4fVe9k9khYdnDoFu46VsA9KCwb9HOlj3dJrA/nA805pPwJrPJT9DNYwIfX8LTMUJsc/z2+O900d23avh3z7gBn+fu/8KTNUJqwzyB90n5W5n+5z1DfWPRiE0n4LpXsGvjxxraYq2naXEWSNMdnAdlz3TRf3fA5rsM4GOpajzCpNRBrgut0et81hLSX3ly/fO3/KrJJEJExEIkSkiYg8AbQDXnLM1n3mgYi0Ap4E7jDGnPCQJWT2WygFg/qU/dS0mqpo20vbP/Xd8vqyH/0ps8oSEQHexfqu/9ORrPvLsxlY16H3A3cDfzbGfO+Yp/vMjeO79R7W2dN/S8kWMvstlIIBnN4T12qC0vaBe7o/+9HXMquqF7Aur91ijNnoNk/3l6sHgd7AGKxGBTNE5Cq3PLrPTrkJOAv4iw95q/x+C6VgUN4nrtUEaY7X0vbPMbe8vuxHf8qskkTkGazruXcZY6Y5zdL95YExZocxZrkxZrYx5irgB+ANsR5spfvMiYgkAc8DzwInRSTe0awzAgh3fK5FCO23UAoG5XriWg2x3vHqcq1QRGoDrXHdN+vd8zl0xWrCtqkcZVY5IvIk8DDwoDHmVbfZHrfNoSsl95cv3zt/ygwVy4AEoAG6z9w1w3qQ/bNYLdWKpn5Y23McmEwo7bdg34n34479hVinPxe7pS8ANgW7fpW0D0prTRSB1Qx0llv6BEd+5/b1tzvSznRKi8K6Kfx9ecqsahPwuKOOj3rJsxz4HdemeUMp2bbe5++dr2WGwoTVpPEnrINahO6zEvunLlZbf/fpD2Cb432rUNpvQd+pfn455wFHgRuxOkFNA+zARcGuX4C3/XLH9A/HH/txx+eRTnnGO+a97vgi3gqcAD53K6sWsAGrX8EVWG2Tv8HqMNPLLa9PZValCeuykAFmA33cph5O+YZgNaX9zPFPdC1wAKu9eHh5vne+llnVJuBjrAdXXYbVUe9KTnU6u0P3mV/7cj4l+xmExH4L+s7zc0fHOg5MB7F6lK4ELgl2vSphu00p0y63fNdgNS3Lw2oR8jwQ46G8RlhPnjsGZGN1aulfyrp9KrOqTI5/Rl/31wXAb47v0hGsliEJp/O987XMqjQBdwBLsK5FFzpef3A/AOk+8/n794eH9Cq/3/R5BkoppULqBrJSSqkA0WCglFJKg4FSSikNBkoppdBgoJRSCg0GSiml0GCglFIKDQZKKaXQYKCUUgr4f1qKztrz2BdEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 0 with valid_loss value: 2.071021556854248.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/skumar/miniconda3/envs/ImageCaption/lib/python3.7/site-packages/ipykernel_launcher.py:34: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 1 with valid_loss value: 1.855028748512268.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/skumar/miniconda3/envs/ImageCaption/lib/python3.7/site-packages/ipykernel_launcher.py:34: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 2 with valid_loss value: 1.7218936681747437.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/skumar/miniconda3/envs/ImageCaption/lib/python3.7/site-packages/ipykernel_launcher.py:34: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 3 with valid_loss value: 1.6429928541183472.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/skumar/miniconda3/envs/ImageCaption/lib/python3.7/site-packages/ipykernel_launcher.py:34: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 4 with valid_loss value: 1.608099102973938.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/skumar/miniconda3/envs/ImageCaption/lib/python3.7/site-packages/ipykernel_launcher.py:34: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 5 with valid_loss value: 1.5958151817321777.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/skumar/miniconda3/envs/ImageCaption/lib/python3.7/site-packages/ipykernel_launcher.py:34: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "/home/skumar/miniconda3/envs/ImageCaption/lib/python3.7/site-packages/ipykernel_launcher.py:34: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "/home/skumar/miniconda3/envs/ImageCaption/lib/python3.7/site-packages/ipykernel_launcher.py:34: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "/home/skumar/miniconda3/envs/ImageCaption/lib/python3.7/site-packages/ipykernel_launcher.py:34: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    }
   ],
   "source": [
    "from torch.utils.tensorboard import SummaryWriter\n",
    "from fastai.callbacks.TensorBoardCallback import *\n",
    "from fastai.callbacks import lr_finder, SaveModelCallback\n",
    "\n",
    "\n",
    "writer = SummaryWriter(comment='stage1')\n",
    "tb_callback = partial(TensorBoardFastAI, writer, \n",
    "                      track_weight=True, track_grad=True, \n",
    "                      metric_names=['val loss','accuracy','bleu_metric']) #beam_search_bleu4\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "learn_LSL.fit_one_cycle(10, 5e-4, div_factor=5,callbacks = [SaveModelCallback(learn_LSL, monitor='valid_loss',name='bestmodel'),\n",
    "                                                            tb_callback(learn_LSL)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-02T06:34:19.987681Z",
     "start_time": "2020-07-02T06:34:19.500241Z"
    }
   },
   "outputs": [],
   "source": [
    "learn_LSL.load('bestmodel');\n",
    "#learn_LSL.export('Label_smoothing_trnsformer.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-02T08:22:27.167064Z",
     "start_time": "2020-07-02T08:22:27.135547Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_predictions(learn, ds_type=DatasetType.Valid):\n",
    "    learn.model.eval()\n",
    "    inputs, targets, outputs = [],[],[]\n",
    "    with torch.no_grad():\n",
    "        for xb,yb in progress_bar(learn.dl(ds_type)):\n",
    "            out = learn.model(*xb)[0]\n",
    "            for x,y,z in zip(xb[0],yb,out):\n",
    "                inputs.append(learn.data.train_ds.x.reconstruct(x.cpu()))\n",
    "                targets.append(learn.data.train_ds.y.reconstruct(y.cpu()))\n",
    "                outputs.append(learn.data.train_ds.y.reconstruct(z.cpu().argmax(1)))\n",
    "    return inputs, targets, outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-02T08:22:40.135286Z",
     "start_time": "2020-07-02T08:22:27.728178Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='109' class='' max='109', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [109/109 00:12<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "inputs, targets, outputs = get_predictions(learn_LSL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-02T08:22:41.185773Z",
     "start_time": "2020-07-02T08:22:41.159518Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Text si on parle des gens en xxunk , xxunk que la dynamique de travail et xxunk est différente de celle d’une installation permanente ?,\n",
       " Text when it comes to the people on tour , i xxunk that the xxunk dynamic is different from what goes on in a permanent production ?,\n",
       " Text when people comes to people people , people , if think , there workplace of is no than any they into a one permanent one of)"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs[500],targets[500],outputs[500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-22T18:41:22.697679Z",
     "start_time": "2020-06-22T18:41:22.508718Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shutdown scheduled for Tue 2020-06-23 00:21:22 IST, use 'shutdown -c' to cancel.\r\n"
     ]
    }
   ],
   "source": [
    "!shutdown -h 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-02T08:22:52.255034Z",
     "start_time": "2020-07-02T08:22:52.223169Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Text quels sont les prix de vente en gros et au détail que vous prévoyez demander pour votre ( vos ) xxunk ) ou produit xxunk contenant notre matériel ?,\n",
       " Text what will be your anticipated xxunk and retail price for each of your xxunk ) or any work derived from using our material ?,\n",
       " Text which price be the organization rate price you development would you product your product ( or your technical ( ( your your product ()"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs[22],targets[22],outputs[22]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-02T08:22:55.685939Z",
     "start_time": "2020-07-02T08:22:55.655206Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Text quelles sont les possibilités qui s’offrent au canada au modèle de coopérative de santé comme organisme susceptible de contribuer à la prestation fructueuse des services de santé primaires ?,\n",
       " Text what are the possibilities in canada for the health co - operative model of organization to contribute to the successful delivery of primary healthcare services ?,\n",
       " Text which possibilities canada opportunities for canada ’s health model care - organization health of health as achieve to health primary services of health health services offered)"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs[0],targets[0],outputs[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Deplotment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-02T08:23:30.207213Z",
     "start_time": "2020-07-02T08:23:29.567200Z"
    }
   },
   "outputs": [],
   "source": [
    "learn_LSL.model.eval()\n",
    "text = \"qu'est-ce que le gouvernement responsable?\"\n",
    "enc_inp = torch.tensor(databunch.x.process_one(text)).unsqueeze(0)\n",
    "dec_out = [2]\n",
    "dec_len = torch.tensor([50])\n",
    "\n",
    "while len(dec_out) < 50:\n",
    "    dec_input = torch.as_tensor([dec_out], dtype=torch.long)\n",
    "    enc_inp,dec_input,dec_len=[i.to('cuda') for i in (enc_inp,dec_input,dec_len)]\n",
    "    with torch.no_grad():\n",
    "        out = learn_LSL.model(enc_inp[:,1:],dec_input,dec_len)\n",
    "    predict = out[0].squeeze(0).argmax(1)[-1].detach().cpu().numpy()\n",
    "    dec_out.append(int(predict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-02T08:23:30.789070Z",
     "start_time": "2020-07-02T08:23:30.758834Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text xxbos where is government ?"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn_LSL.data.train_ds.y.reconstruct(torch.as_tensor([dec_out], dtype=torch.long)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-02T07:19:35.508301Z",
     "start_time": "2020-07-02T07:19:35.476008Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1]])"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.as_tensor([dec_out], dtype=torch.long)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-24T14:46:42.466363Z",
     "start_time": "2020-05-24T14:46:42.301307Z"
    }
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "can't convert CUDA tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-77-8376eda77ed3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mout_text\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_ds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreconstruct\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/miniconda3/envs/ImageCaption/lib/python3.7/site-packages/fastai/text/data.py\u001b[0m in \u001b[0;36mreconstruct\u001b[0;34m(self, t)\u001b[0m\n\u001b[1;32m    339\u001b[0m         \u001b[0midx_min\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpad_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnonzero\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    340\u001b[0m         \u001b[0midx_max\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpad_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnonzero\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 341\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mText\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx_min\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0midx_max\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocab\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtextify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx_min\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0midx_max\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    342\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    343\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mclassmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/ImageCaption/lib/python3.7/site-packages/fastai/text/data.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, ids, text)\u001b[0m\n\u001b[1;32m    278\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mText\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mItemBase\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    279\u001b[0m     \u001b[0;34m\"Basic item for <code>text</code> data in numericalized `ids`.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 280\u001b[0;31m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mint64\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    281\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__str__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;32mreturn\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    282\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/ImageCaption/lib/python3.7/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36m__array__\u001b[0;34m(self, dtype)\u001b[0m\n\u001b[1;32m    486\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    487\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 488\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    489\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    490\u001b[0m     \u001b[0;31m# Wrap Numpy array again in a suitable tensor when done, to support e.g.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: can't convert CUDA tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first."
     ]
    }
   ],
   "source": [
    "out[0].squeeze(0).argmax(1)\n",
    "out_text = learn.data.train_ds.y.reconstruct(out[0].squeeze(0).argmax(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-07-02T08:23:40.635996Z",
     "start_time": "2020-07-02T08:23:40.606472Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Text quelles sont les priorités relatives des différents projets , existe - il une relation xxunk , et ces priorités doivent - elles toutes être incluses dans le xxunk ?,\n",
       " Text what are the relative priorities of the various projects , is there a xxunk relationship , and should they all be included in the xxunk ?,\n",
       " Text which priorities the priorities projects that projects projects projects , including needed any priorities , between and what be be these included in regard relationship ))"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs[21], targets[21], outputs[21]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Initialization Cell",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "notify_time": "5",
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
